Open-sourcing Representation Manager
Representation Manager (RMS) serves as a centralized embedding management system, providing SimClusters or other embeddings as facade of the underlying storage or services.
This commit is contained in:
twitter-team
parent
197bf2c563
commit
43cdcf2ed6
|
|
@ -0,0 +1 @@
|
|||
# This prevents SQ query from grabbing //:all since it traverses up once to find a BUILD
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
# Representation Manager #
|
||||
|
||||
**Representation Manager** (RMS) serves as a centralized embedding management system, providing SimClusters or other embeddings as facade of the underlying storage or services.
|
||||
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
JOB=representation-manager bazel run --ui_event_filters=-info,-stdout,-stderr --noshow_progress \
|
||||
//relevance-platform/src/main/python/deploy -- "$@"
|
||||
|
|
@ -0,0 +1,17 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
|
||||
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/readablestore",
|
||||
"representation-manager/client/src/main/scala/com/twitter/representation_manager/config",
|
||||
"representation-manager/server/src/main/thrift:thrift-scala",
|
||||
"src/scala/com/twitter/simclusters_v2/common",
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
|
||||
"stitch/stitch-storehaus",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,208 @@
|
|||
package com.twitter.representation_manager
|
||||
|
||||
import com.twitter.finagle.memcached.{Client => MemcachedClient}
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.store.strato.StratoFetchableStore
|
||||
import com.twitter.hermit.store.common.ObservedCachedReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedReadableStore
|
||||
import com.twitter.representation_manager.config.ClientConfig
|
||||
import com.twitter.representation_manager.config.DisabledInMemoryCacheParams
|
||||
import com.twitter.representation_manager.config.EnabledInMemoryCacheParams
|
||||
import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.LocaleEntityId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.TopicId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
|
||||
/**
|
||||
* This is the class that offers features to build readable stores for a given
|
||||
* SimClustersEmbeddingView (i.e. embeddingType and modelVersion). It applies ClientConfig
|
||||
* for a particular service and build ReadableStores which implement that config.
|
||||
*/
|
||||
class StoreBuilder(
|
||||
clientConfig: ClientConfig,
|
||||
stratoClient: StratoClient,
|
||||
memCachedClient: MemcachedClient,
|
||||
globalStats: StatsReceiver,
|
||||
) {
|
||||
private val stats =
|
||||
globalStats.scope("representation_manager_client").scope(this.getClass.getSimpleName)
|
||||
|
||||
// Column consts
|
||||
private val ColPathPrefix = "recommendations/representation_manager/"
|
||||
private val SimclustersTweetColPath = ColPathPrefix + "simClustersEmbedding.Tweet"
|
||||
private val SimclustersUserColPath = ColPathPrefix + "simClustersEmbedding.User"
|
||||
private val SimclustersTopicIdColPath = ColPathPrefix + "simClustersEmbedding.TopicId"
|
||||
private val SimclustersLocaleEntityIdColPath =
|
||||
ColPathPrefix + "simClustersEmbedding.LocaleEntityId"
|
||||
|
||||
def buildSimclustersTweetEmbeddingStore(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[Long, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersTweetColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
addCacheLayer(rawStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersUserEmbeddingStore(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[Long, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersUserColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
addCacheLayer(rawStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersTopicIdEmbeddingStore(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[TopicId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersTopicIdColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
addCacheLayer(rawStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersLocaleEntityIdEmbeddingStore(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[LocaleEntityId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[LocaleEntityId, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersLocaleEntityIdColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
addCacheLayer(rawStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersTweetEmbeddingStoreWithEmbeddingIdAsKey(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersTweetColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.TweetId(tweetId)) =>
|
||||
tweetId
|
||||
}
|
||||
|
||||
addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersUserEmbeddingStoreWithEmbeddingIdAsKey(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[Long, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersUserColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.UserId(userId)) =>
|
||||
userId
|
||||
}
|
||||
|
||||
addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersTopicEmbeddingStoreWithEmbeddingIdAsKey(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersTopicIdColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.TopicId(topicId)) =>
|
||||
topicId
|
||||
}
|
||||
|
||||
addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersTopicIdEmbeddingStoreWithEmbeddingIdAsKey(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[TopicId, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersTopicIdColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.TopicId(topicId)) =>
|
||||
topicId
|
||||
}
|
||||
|
||||
addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
def buildSimclustersLocaleEntityIdEmbeddingStoreWithEmbeddingIdAsKey(
|
||||
embeddingColumnView: SimClustersEmbeddingView
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withView[LocaleEntityId, SimClustersEmbeddingView, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
SimclustersLocaleEntityIdColPath,
|
||||
embeddingColumnView)
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
val embeddingIdAsKeyStore = rawStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.LocaleEntityId(localeEntityId)) =>
|
||||
localeEntityId
|
||||
}
|
||||
|
||||
addCacheLayer(embeddingIdAsKeyStore, embeddingColumnView)
|
||||
}
|
||||
|
||||
private def addCacheLayer[K](
|
||||
rawStore: ReadableStore[K, SimClustersEmbedding],
|
||||
embeddingColumnView: SimClustersEmbeddingView,
|
||||
): ReadableStore[K, SimClustersEmbedding] = {
|
||||
// Add in-memory caching based on ClientConfig
|
||||
val inMemCacheParams = clientConfig.inMemoryCacheConfig
|
||||
.getCacheSetup(embeddingColumnView.embeddingType, embeddingColumnView.modelVersion)
|
||||
|
||||
val statsPerStore = stats
|
||||
.scope(embeddingColumnView.embeddingType.name).scope(embeddingColumnView.modelVersion.name)
|
||||
|
||||
inMemCacheParams match {
|
||||
case DisabledInMemoryCacheParams =>
|
||||
ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(statsPerStore)
|
||||
case EnabledInMemoryCacheParams(ttl, maxKeys, cacheName) =>
|
||||
ObservedCachedReadableStore.from[K, SimClustersEmbedding](
|
||||
rawStore,
|
||||
ttl = ttl,
|
||||
maxKeys = maxKeys,
|
||||
cacheName = cacheName,
|
||||
windowSize = 10000L
|
||||
)(statsPerStore)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,12 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/common",
|
||||
"representation-manager/server/src/main/thrift:thrift-scala",
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package com.twitter.representation_manager.config
|
||||
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
|
||||
/*
|
||||
* This is RMS client config class.
|
||||
* We only support setting up in memory cache params for now, but we expect to enable other
|
||||
* customisations in the near future e.g. request timeout
|
||||
*
|
||||
* --------------------------------------------
|
||||
* PLEASE NOTE:
|
||||
* Having in-memory cache is not necessarily a free performance win, anyone considering it should
|
||||
* investigate rather than blindly enabling it
|
||||
* */
|
||||
class ClientConfig(inMemCacheParamsOverrides: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
InMemoryCacheParams
|
||||
] = Map.empty) {
|
||||
// In memory cache config per embedding
|
||||
val inMemCacheParams = DefaultInMemoryCacheConfig.cacheParamsMap ++ inMemCacheParamsOverrides
|
||||
val inMemoryCacheConfig = new InMemoryCacheConfig(inMemCacheParams)
|
||||
}
|
||||
|
||||
object DefaultClientConfig extends ClientConfig
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
package com.twitter.representation_manager.config
|
||||
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/*
|
||||
* --------------------------------------------
|
||||
* PLEASE NOTE:
|
||||
* Having in-memory cache is not necessarily a free performance win, anyone considering it should
|
||||
* investigate rather than blindly enabling it
|
||||
* --------------------------------------------
|
||||
* */
|
||||
|
||||
sealed trait InMemoryCacheParams
|
||||
|
||||
/*
|
||||
* This holds params that is required to set up a in-mem cache for a single embedding store
|
||||
*/
|
||||
case class EnabledInMemoryCacheParams(
|
||||
ttl: Duration,
|
||||
maxKeys: Int,
|
||||
cacheName: String)
|
||||
extends InMemoryCacheParams
|
||||
object DisabledInMemoryCacheParams extends InMemoryCacheParams
|
||||
|
||||
/*
|
||||
* This is the class for the in-memory cache config. Client could pass in their own cacheParamsMap to
|
||||
* create a new InMemoryCacheConfig instead of using the DefaultInMemoryCacheConfig object below
|
||||
* */
|
||||
class InMemoryCacheConfig(
|
||||
cacheParamsMap: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
InMemoryCacheParams
|
||||
] = Map.empty) {
|
||||
|
||||
def getCacheSetup(
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): InMemoryCacheParams = {
|
||||
// When requested embedding type doesn't exist, we return DisabledInMemoryCacheParams
|
||||
cacheParamsMap.getOrElse((embeddingType, modelVersion), DisabledInMemoryCacheParams)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Default config for the in-memory cache
|
||||
* Clients can directly import and use this one if they don't want to set up a customised config
|
||||
* */
|
||||
object DefaultInMemoryCacheConfig extends InMemoryCacheConfig {
|
||||
// set default to no in-memory caching
|
||||
val cacheParamsMap = Map.empty
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
jvm_binary(
|
||||
name = "bin",
|
||||
basename = "representation-manager",
|
||||
main = "com.twitter.representation_manager.RepresentationManagerFedServerMain",
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-logback/src/main/scala",
|
||||
"loglens/loglens-logback/src/main/scala/com/twitter/loglens/logback",
|
||||
"representation-manager/server/src/main/resources",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager",
|
||||
"twitter-server/logback-classic/src/main/scala",
|
||||
],
|
||||
)
|
||||
|
||||
# Aurora Workflows build phase convention requires a jvm_app named with ${project-name}-app
|
||||
jvm_app(
|
||||
name = "representation-manager-app",
|
||||
archive = "zip",
|
||||
binary = ":bin",
|
||||
)
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
resources(
|
||||
sources = [
|
||||
"*.xml",
|
||||
"config/*.yml",
|
||||
],
|
||||
tags = ["bazel-compatible"],
|
||||
)
|
||||
|
|
@ -0,0 +1,219 @@
|
|||
# ---------- traffic percentage by embedding type and model version ----------
|
||||
# Decider strings are build dynamically following the rule in there
|
||||
# i.e. s"enable_${embeddingType.name}_${modelVersion.name}"
|
||||
# Hence this should be updated accordingly if usage is changed in the embedding stores
|
||||
|
||||
# Tweet embeddings
|
||||
"enable_LogFavBasedTweet_Model20m145k2020":
|
||||
comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavBasedTweet - Model20m145k2020. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedTweet_Model20m145kUpdated":
|
||||
comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavBasedTweet - Model20m145kUpdated. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavLongestL2EmbeddingTweet_Model20m145k2020":
|
||||
comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavLongestL2EmbeddingTweet - Model20m145k2020. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavLongestL2EmbeddingTweet_Model20m145kUpdated":
|
||||
comment: "Enable x% read traffic (0<=x<=10000, e.g. 1000=10%) for LogFavLongestL2EmbeddingTweet - Model20m145kUpdated. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
# Topic embeddings
|
||||
"enable_FavTfgTopic_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FavTfgTopic - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedKgoApeTopic_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedKgoApeTopic - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
# User embeddings - KnownFor
|
||||
"enable_FavBasedProducer_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to FavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FavBasedProducer_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FollowBasedProducer_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FollowBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_AggregatableFavBasedProducer_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to AggregatableFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_AggregatableFavBasedProducer_Model20m145k2020":
|
||||
comment: "Enable the read traffic to AggregatableFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_AggregatableLogFavBasedProducer_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to AggregatableLogFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_AggregatableLogFavBasedProducer_Model20m145k2020":
|
||||
comment: "Enable the read traffic to AggregatableLogFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
enable_RelaxedAggregatableLogFavBasedProducer_Model20m145kUpdated:
|
||||
comment: "Enable the read traffic to RelaxedAggregatableLogFavBasedProducer - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
enable_RelaxedAggregatableLogFavBasedProducer_Model20m145k2020:
|
||||
comment: "Enable the read traffic to RelaxedAggregatableLogFavBasedProducer - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
# User embeddings - InterestedIn
|
||||
"enable_LogFavBasedUserInterestedInFromAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedInFromAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FollowBasedUserInterestedInFromAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FollowBasedUserInterestedInFromAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FavBasedUserInterestedIn_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to FavBasedUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FavBasedUserInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FavBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FollowBasedUserInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FollowBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FavBasedUserInterestedInFromPE_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to FavBasedUserInterestedInFromPE - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FilteredUserInterestedIn_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to FilteredUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FilteredUserInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to FilteredUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_FilteredUserInterestedInFromPE_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to FilteredUserInterestedInFromPE - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_UnfilteredUserInterestedIn_Model20m145kUpdated":
|
||||
comment: "Enable the read traffic to UnfilteredUserInterestedIn - Model20m145kUpdated from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_UnfilteredUserInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to UnfilteredUserInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_UserNextInterestedIn_Model20m145k2020":
|
||||
comment: "Enable the read traffic to UserNextInterestedIn - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedAverageAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
"enable_LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE_Model20m145k2020":
|
||||
comment: "Enable the read traffic to LogFavBasedUserInterestedAverageAddressBookFromIIAPE - Model20m145k2020 from 0% to 100%. 0 means return EMPTY for all requests."
|
||||
default_availability: 10000
|
||||
|
||||
# ---------- load shedding by caller id ----------
|
||||
# To create a new decider, add here with the same format and caller's details :
|
||||
# "representation-manager_load_shed_by_caller_id_twtr:{{role}}:{{name}}:{{environment}}:{{cluster}}"
|
||||
# All the deciders below are generated by this script:
|
||||
# ./strato/bin/fed deciders representation-manager --service-role=representation-manager --service-name=representation-manager
|
||||
# If you need to run the script and paste the output, add ONLY the prod deciders here.
|
||||
"representation-manager_load_shed_by_caller_id_all":
|
||||
comment: "Reject all traffic from caller id: all"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:cr-mixer:cr-mixer:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:cr-mixer:cr-mixer:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:cr-mixer:cr-mixer:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:cr-mixer:cr-mixer:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-1:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-1:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-1:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-1:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-3:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-3:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-3:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-3:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-4:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-4:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-4:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-4:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann-experimental:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:simclusters-ann:simclusters-ann:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:simclusters-ann:simclusters-ann:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoapi:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoapi:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoserver:prod:atla":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoserver:prod:atla"
|
||||
default_availability: 0
|
||||
|
||||
"representation-manager_load_shed_by_caller_id_twtr:svc:stratostore:stratoserver:prod:pdxa":
|
||||
comment: "Reject all traffic from caller id: twtr:svc:stratostore:stratoserver:prod:pdxa"
|
||||
default_availability: 0
|
||||
|
||||
# ---------- Dark Traffic Proxy ----------
|
||||
representation-manager_forward_dark_traffic:
|
||||
comment: "Defines the percentage of traffic to forward to diffy-proxy. Set to 0 to disable dark traffic forwarding"
|
||||
default_availability: 0
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
<configuration>
|
||||
<shutdownHook class="ch.qos.logback.core.hook.DelayingShutdownHook"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Service Config -->
|
||||
<!-- ===================================================== -->
|
||||
<property name="DEFAULT_SERVICE_PATTERN"
|
||||
value="%-16X{traceId} %-12X{clientId:--} %-16X{method} %-25logger{0} %msg"/>
|
||||
|
||||
<property name="DEFAULT_ACCESS_PATTERN"
|
||||
value="%msg"/>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Common Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- JUL/JDK14 to Logback bridge -->
|
||||
<contextListener class="ch.qos.logback.classic.jul.LevelChangePropagator">
|
||||
<resetJUL>true</resetJUL>
|
||||
</contextListener>
|
||||
|
||||
<!-- ====================================================================================== -->
|
||||
<!-- NOTE: The following appenders use a simple TimeBasedRollingPolicy configuration. -->
|
||||
<!-- You may want to consider using a more advanced SizeAndTimeBasedRollingPolicy. -->
|
||||
<!-- See: https://logback.qos.ch/manual/appenders.html#SizeAndTimeBasedRollingPolicy -->
|
||||
<!-- ====================================================================================== -->
|
||||
|
||||
<!-- Service Log (rollover daily, keep maximum of 21 days of gzip compressed logs) -->
|
||||
<appender name="SERVICE" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.service.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${log.service.output}.%d.gz</fileNamePattern>
|
||||
<!-- the maximum total size of all the log files -->
|
||||
<totalSizeCap>3GB</totalSizeCap>
|
||||
<!-- keep maximum 21 days' worth of history -->
|
||||
<maxHistory>21</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- Access Log (rollover daily, keep maximum of 21 days of gzip compressed logs) -->
|
||||
<appender name="ACCESS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>${log.access.output}</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>${log.access.output}.%d.gz</fileNamePattern>
|
||||
<!-- the maximum total size of all the log files -->
|
||||
<totalSizeCap>100MB</totalSizeCap>
|
||||
<!-- keep maximum 7 days' worth of history -->
|
||||
<maxHistory>7</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>${DEFAULT_ACCESS_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!--LogLens -->
|
||||
<appender name="LOGLENS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/service</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- LogLens Access -->
|
||||
<appender name="LOGLENS-ACCESS" class="com.twitter.loglens.logback.LoglensAppender">
|
||||
<mdcAdditionalContext>true</mdcAdditionalContext>
|
||||
<category>${log.lens.category}</category>
|
||||
<index>${log.lens.index}</index>
|
||||
<tag>${log.lens.tag}/access</tag>
|
||||
<encoder>
|
||||
<pattern>%msg</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- Pipeline Execution Logs -->
|
||||
<appender name="ALLOW-LISTED-PIPELINE-EXECUTIONS" class="ch.qos.logback.core.rolling.RollingFileAppender">
|
||||
<file>allow_listed_pipeline_executions.log</file>
|
||||
<rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
|
||||
<!-- daily rollover -->
|
||||
<fileNamePattern>allow_listed_pipeline_executions.log.%d.gz</fileNamePattern>
|
||||
<!-- the maximum total size of all the log files -->
|
||||
<totalSizeCap>100MB</totalSizeCap>
|
||||
<!-- keep maximum 7 days' worth of history -->
|
||||
<maxHistory>7</maxHistory>
|
||||
<cleanHistoryOnStart>true</cleanHistoryOnStart>
|
||||
</rollingPolicy>
|
||||
<encoder>
|
||||
<pattern>%date %.-3level ${DEFAULT_SERVICE_PATTERN}%n</pattern>
|
||||
</encoder>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Primary Async Appenders -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<property name="async_queue_size" value="${queue.size:-50000}"/>
|
||||
<property name="async_max_flush_time" value="${max.flush.time:-0}"/>
|
||||
|
||||
<appender name="ASYNC-SERVICE" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="SERVICE"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-ALLOW-LISTED-PIPELINE-EXECUTIONS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="ALLOW-LISTED-PIPELINE-EXECUTIONS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS"/>
|
||||
</appender>
|
||||
|
||||
<appender name="ASYNC-LOGLENS-ACCESS" class="com.twitter.inject.logback.AsyncAppender">
|
||||
<queueSize>${async_queue_size}</queueSize>
|
||||
<maxFlushTime>${async_max_flush_time}</maxFlushTime>
|
||||
<appender-ref ref="LOGLENS-ACCESS"/>
|
||||
</appender>
|
||||
|
||||
<!-- ===================================================== -->
|
||||
<!-- Package Config -->
|
||||
<!-- ===================================================== -->
|
||||
|
||||
<!-- Per-Package Config -->
|
||||
<logger name="com.twitter" level="INHERITED"/>
|
||||
<logger name="com.twitter.wilyns" level="INHERITED"/>
|
||||
<logger name="com.twitter.configbus.client.file" level="INHERITED"/>
|
||||
<logger name="com.twitter.finagle.mux" level="INHERITED"/>
|
||||
<logger name="com.twitter.finagle.serverset2" level="INHERITED"/>
|
||||
<logger name="com.twitter.logging.ScribeHandler" level="INHERITED"/>
|
||||
<logger name="com.twitter.zookeeper.client.internal" level="INHERITED"/>
|
||||
|
||||
<!-- Root Config -->
|
||||
<!-- For all logs except access logs, disable logging below log_level level by default. This can be overriden in the per-package loggers, and dynamically in the admin panel of individual instances. -->
|
||||
<root level="${log_level:-INFO}">
|
||||
<appender-ref ref="ASYNC-SERVICE"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS"/>
|
||||
</root>
|
||||
|
||||
<!-- Access Logging -->
|
||||
<!-- Access logs are turned off by default -->
|
||||
<logger name="com.twitter.finatra.thrift.filters.AccessLoggingFilter" level="OFF" additivity="false">
|
||||
<appender-ref ref="ASYNC-ACCESS"/>
|
||||
<appender-ref ref="ASYNC-LOGLENS-ACCESS"/>
|
||||
</logger>
|
||||
|
||||
</configuration>
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-thrift-client",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/topic",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/tweet",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns/user",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.twitter.representation_manager
|
||||
|
||||
import com.google.inject.Module
|
||||
import com.twitter.inject.thrift.modules.ThriftClientIdModule
|
||||
import com.twitter.representation_manager.columns.topic.LocaleEntityIdSimClustersEmbeddingCol
|
||||
import com.twitter.representation_manager.columns.topic.TopicIdSimClustersEmbeddingCol
|
||||
import com.twitter.representation_manager.columns.tweet.TweetSimClustersEmbeddingCol
|
||||
import com.twitter.representation_manager.columns.user.UserSimClustersEmbeddingCol
|
||||
import com.twitter.representation_manager.modules.CacheModule
|
||||
import com.twitter.representation_manager.modules.InterestsThriftClientModule
|
||||
import com.twitter.representation_manager.modules.LegacyRMSConfigModule
|
||||
import com.twitter.representation_manager.modules.StoreModule
|
||||
import com.twitter.representation_manager.modules.TimerModule
|
||||
import com.twitter.representation_manager.modules.UttClientModule
|
||||
import com.twitter.strato.fed._
|
||||
import com.twitter.strato.fed.server._
|
||||
|
||||
object RepresentationManagerFedServerMain extends RepresentationManagerFedServer
|
||||
|
||||
trait RepresentationManagerFedServer extends StratoFedServer {
|
||||
override def dest: String = "/s/representation-manager/representation-manager"
|
||||
override val modules: Seq[Module] =
|
||||
Seq(
|
||||
CacheModule,
|
||||
InterestsThriftClientModule,
|
||||
LegacyRMSConfigModule,
|
||||
StoreModule,
|
||||
ThriftClientIdModule,
|
||||
TimerModule,
|
||||
UttClientModule
|
||||
)
|
||||
|
||||
override def columns: Seq[Class[_ <: StratoFed.Column]] =
|
||||
Seq(
|
||||
classOf[TweetSimClustersEmbeddingCol],
|
||||
classOf[UserSimClustersEmbeddingCol],
|
||||
classOf[TopicIdSimClustersEmbeddingCol],
|
||||
classOf[LocaleEntityIdSimClustersEmbeddingCol]
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,9 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
package com.twitter.representation_manager.columns
|
||||
|
||||
import com.twitter.strato.access.Access.LdapGroup
|
||||
import com.twitter.strato.config.ContactInfo
|
||||
import com.twitter.strato.config.FromColumns
|
||||
import com.twitter.strato.config.Has
|
||||
import com.twitter.strato.config.Prefix
|
||||
import com.twitter.strato.config.ServiceIdentifierPattern
|
||||
|
||||
object ColumnConfigBase {
|
||||
|
||||
/****************** Internal permissions *******************/
|
||||
val recosPermissions: Seq[com.twitter.strato.config.Policy] = Seq()
|
||||
|
||||
/****************** External permissions *******************/
|
||||
// This is used to grant limited access to members outside of RP team.
|
||||
val externalPermissions: Seq[com.twitter.strato.config.Policy] = Seq()
|
||||
|
||||
val contactInfo: ContactInfo = ContactInfo(
|
||||
description = "Please contact Relevance Platform for more details",
|
||||
contactEmail = "no-reply@twitter.com",
|
||||
ldapGroup = "ldap",
|
||||
jiraProject = "JIRA",
|
||||
links = Seq("http://go/rms-runbook")
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/modules",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/store",
|
||||
"representation-manager/server/src/main/thrift:thrift-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
package com.twitter.representation_manager.columns.topic
|
||||
|
||||
import com.twitter.representation_manager.columns.ColumnConfigBase
|
||||
import com.twitter.representation_manager.store.TopicSimClustersEmbeddingStore
|
||||
import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.LocaleEntityId
|
||||
import com.twitter.stitch
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.storehaus.StitchOfReadableStore
|
||||
import com.twitter.strato.catalog.OpMetadata
|
||||
import com.twitter.strato.config.AnyOf
|
||||
import com.twitter.strato.config.ContactInfo
|
||||
import com.twitter.strato.config.FromColumns
|
||||
import com.twitter.strato.config.Policy
|
||||
import com.twitter.strato.config.Prefix
|
||||
import com.twitter.strato.data.Conv
|
||||
import com.twitter.strato.data.Description.PlainText
|
||||
import com.twitter.strato.data.Lifecycle
|
||||
import com.twitter.strato.fed._
|
||||
import com.twitter.strato.thrift.ScroogeConv
|
||||
import javax.inject.Inject
|
||||
|
||||
class LocaleEntityIdSimClustersEmbeddingCol @Inject() (
|
||||
embeddingStore: TopicSimClustersEmbeddingStore)
|
||||
extends StratoFed.Column(
|
||||
"recommendations/representation_manager/simClustersEmbedding.LocaleEntityId")
|
||||
with StratoFed.Fetch.Stitch {
|
||||
|
||||
private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] =
|
||||
StitchOfReadableStore(embeddingStore.topicSimClustersEmbeddingStore.mapValues(_.toThrift))
|
||||
|
||||
val colPermissions: Seq[com.twitter.strato.config.Policy] =
|
||||
ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns(
|
||||
Set(
|
||||
Prefix("ml/featureStore/simClusters"),
|
||||
))
|
||||
|
||||
override val policy: Policy = AnyOf({
|
||||
colPermissions
|
||||
})
|
||||
|
||||
override type Key = LocaleEntityId
|
||||
override type View = SimClustersEmbeddingView
|
||||
override type Value = SimClustersEmbedding
|
||||
|
||||
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[LocaleEntityId]
|
||||
override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView]
|
||||
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding]
|
||||
|
||||
override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo
|
||||
|
||||
override val metadata: OpMetadata = OpMetadata(
|
||||
lifecycle = Some(Lifecycle.Production),
|
||||
description = Some(
|
||||
PlainText(
|
||||
"The Topic SimClusters Embedding Endpoint in Representation Management Service with LocaleEntityId." +
|
||||
" TDD: http://go/rms-tdd"))
|
||||
)
|
||||
|
||||
override def fetch(key: Key, view: View): Stitch[Result[Value]] = {
|
||||
val embeddingId = SimClustersEmbeddingId(
|
||||
view.embeddingType,
|
||||
view.modelVersion,
|
||||
InternalId.LocaleEntityId(key)
|
||||
)
|
||||
|
||||
storeStitch(embeddingId)
|
||||
.map(embedding => found(embedding))
|
||||
.handle {
|
||||
case stitch.NotFound => missing
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
package com.twitter.representation_manager.columns.topic
|
||||
|
||||
import com.twitter.representation_manager.columns.ColumnConfigBase
|
||||
import com.twitter.representation_manager.store.TopicSimClustersEmbeddingStore
|
||||
import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.TopicId
|
||||
import com.twitter.stitch
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.storehaus.StitchOfReadableStore
|
||||
import com.twitter.strato.catalog.OpMetadata
|
||||
import com.twitter.strato.config.AnyOf
|
||||
import com.twitter.strato.config.ContactInfo
|
||||
import com.twitter.strato.config.FromColumns
|
||||
import com.twitter.strato.config.Policy
|
||||
import com.twitter.strato.config.Prefix
|
||||
import com.twitter.strato.data.Conv
|
||||
import com.twitter.strato.data.Description.PlainText
|
||||
import com.twitter.strato.data.Lifecycle
|
||||
import com.twitter.strato.fed._
|
||||
import com.twitter.strato.thrift.ScroogeConv
|
||||
import javax.inject.Inject
|
||||
|
||||
class TopicIdSimClustersEmbeddingCol @Inject() (embeddingStore: TopicSimClustersEmbeddingStore)
|
||||
extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.TopicId")
|
||||
with StratoFed.Fetch.Stitch {
|
||||
|
||||
private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] =
|
||||
StitchOfReadableStore(embeddingStore.topicSimClustersEmbeddingStore.mapValues(_.toThrift))
|
||||
|
||||
val colPermissions: Seq[com.twitter.strato.config.Policy] =
|
||||
ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns(
|
||||
Set(
|
||||
Prefix("ml/featureStore/simClusters"),
|
||||
))
|
||||
|
||||
override val policy: Policy = AnyOf({
|
||||
colPermissions
|
||||
})
|
||||
|
||||
override type Key = TopicId
|
||||
override type View = SimClustersEmbeddingView
|
||||
override type Value = SimClustersEmbedding
|
||||
|
||||
override val keyConv: Conv[Key] = ScroogeConv.fromStruct[TopicId]
|
||||
override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView]
|
||||
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding]
|
||||
|
||||
override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo
|
||||
|
||||
override val metadata: OpMetadata = OpMetadata(
|
||||
lifecycle = Some(Lifecycle.Production),
|
||||
description = Some(PlainText(
|
||||
"The Topic SimClusters Embedding Endpoint in Representation Management Service with TopicId." +
|
||||
" TDD: http://go/rms-tdd"))
|
||||
)
|
||||
|
||||
override def fetch(key: Key, view: View): Stitch[Result[Value]] = {
|
||||
val embeddingId = SimClustersEmbeddingId(
|
||||
view.embeddingType,
|
||||
view.modelVersion,
|
||||
InternalId.TopicId(key)
|
||||
)
|
||||
|
||||
storeStitch(embeddingId)
|
||||
.map(embedding => found(embedding))
|
||||
.handle {
|
||||
case stitch.NotFound => missing
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/modules",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/store",
|
||||
"representation-manager/server/src/main/thrift:thrift-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package com.twitter.representation_manager.columns.tweet
|
||||
|
||||
import com.twitter.representation_manager.columns.ColumnConfigBase
|
||||
import com.twitter.representation_manager.store.TweetSimClustersEmbeddingStore
|
||||
import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.stitch
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.storehaus.StitchOfReadableStore
|
||||
import com.twitter.strato.catalog.OpMetadata
|
||||
import com.twitter.strato.config.AnyOf
|
||||
import com.twitter.strato.config.ContactInfo
|
||||
import com.twitter.strato.config.FromColumns
|
||||
import com.twitter.strato.config.Policy
|
||||
import com.twitter.strato.config.Prefix
|
||||
import com.twitter.strato.data.Conv
|
||||
import com.twitter.strato.data.Description.PlainText
|
||||
import com.twitter.strato.data.Lifecycle
|
||||
import com.twitter.strato.fed._
|
||||
import com.twitter.strato.thrift.ScroogeConv
|
||||
import javax.inject.Inject
|
||||
|
||||
class TweetSimClustersEmbeddingCol @Inject() (embeddingStore: TweetSimClustersEmbeddingStore)
|
||||
extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.Tweet")
|
||||
with StratoFed.Fetch.Stitch {
|
||||
|
||||
private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] =
|
||||
StitchOfReadableStore(embeddingStore.tweetSimClustersEmbeddingStore.mapValues(_.toThrift))
|
||||
|
||||
val colPermissions: Seq[com.twitter.strato.config.Policy] =
|
||||
ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns(
|
||||
Set(
|
||||
Prefix("ml/featureStore/simClusters"),
|
||||
))
|
||||
|
||||
override val policy: Policy = AnyOf({
|
||||
colPermissions
|
||||
})
|
||||
|
||||
override type Key = Long // TweetId
|
||||
override type View = SimClustersEmbeddingView
|
||||
override type Value = SimClustersEmbedding
|
||||
|
||||
override val keyConv: Conv[Key] = Conv.long
|
||||
override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView]
|
||||
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding]
|
||||
|
||||
override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo
|
||||
|
||||
override val metadata: OpMetadata = OpMetadata(
|
||||
lifecycle = Some(Lifecycle.Production),
|
||||
description = Some(
|
||||
PlainText("The Tweet SimClusters Embedding Endpoint in Representation Management Service." +
|
||||
" TDD: http://go/rms-tdd"))
|
||||
)
|
||||
|
||||
override def fetch(key: Key, view: View): Stitch[Result[Value]] = {
|
||||
val embeddingId = SimClustersEmbeddingId(
|
||||
view.embeddingType,
|
||||
view.modelVersion,
|
||||
InternalId.TweetId(key)
|
||||
)
|
||||
|
||||
storeStitch(embeddingId)
|
||||
.map(embedding => found(embedding))
|
||||
.handle {
|
||||
case stitch.NotFound => missing
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/columns",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/modules",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/store",
|
||||
"representation-manager/server/src/main/thrift:thrift-scala",
|
||||
"strato/src/main/scala/com/twitter/strato/fed",
|
||||
"strato/src/main/scala/com/twitter/strato/fed/server",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
package com.twitter.representation_manager.columns.user
|
||||
|
||||
import com.twitter.representation_manager.columns.ColumnConfigBase
|
||||
import com.twitter.representation_manager.store.UserSimClustersEmbeddingStore
|
||||
import com.twitter.representation_manager.thriftscala.SimClustersEmbeddingView
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.stitch
|
||||
import com.twitter.stitch.Stitch
|
||||
import com.twitter.stitch.storehaus.StitchOfReadableStore
|
||||
import com.twitter.strato.catalog.OpMetadata
|
||||
import com.twitter.strato.config.AnyOf
|
||||
import com.twitter.strato.config.ContactInfo
|
||||
import com.twitter.strato.config.FromColumns
|
||||
import com.twitter.strato.config.Policy
|
||||
import com.twitter.strato.config.Prefix
|
||||
import com.twitter.strato.data.Conv
|
||||
import com.twitter.strato.data.Description.PlainText
|
||||
import com.twitter.strato.data.Lifecycle
|
||||
import com.twitter.strato.fed._
|
||||
import com.twitter.strato.thrift.ScroogeConv
|
||||
import javax.inject.Inject
|
||||
|
||||
class UserSimClustersEmbeddingCol @Inject() (embeddingStore: UserSimClustersEmbeddingStore)
|
||||
extends StratoFed.Column("recommendations/representation_manager/simClustersEmbedding.User")
|
||||
with StratoFed.Fetch.Stitch {
|
||||
|
||||
private val storeStitch: SimClustersEmbeddingId => Stitch[SimClustersEmbedding] =
|
||||
StitchOfReadableStore(embeddingStore.userSimClustersEmbeddingStore.mapValues(_.toThrift))
|
||||
|
||||
val colPermissions: Seq[com.twitter.strato.config.Policy] =
|
||||
ColumnConfigBase.recosPermissions ++ ColumnConfigBase.externalPermissions :+ FromColumns(
|
||||
Set(
|
||||
Prefix("ml/featureStore/simClusters"),
|
||||
))
|
||||
|
||||
override val policy: Policy = AnyOf({
|
||||
colPermissions
|
||||
})
|
||||
|
||||
override type Key = Long // UserId
|
||||
override type View = SimClustersEmbeddingView
|
||||
override type Value = SimClustersEmbedding
|
||||
|
||||
override val keyConv: Conv[Key] = Conv.long
|
||||
override val viewConv: Conv[View] = ScroogeConv.fromStruct[SimClustersEmbeddingView]
|
||||
override val valueConv: Conv[Value] = ScroogeConv.fromStruct[SimClustersEmbedding]
|
||||
|
||||
override val contactInfo: ContactInfo = ColumnConfigBase.contactInfo
|
||||
|
||||
override val metadata: OpMetadata = OpMetadata(
|
||||
lifecycle = Some(Lifecycle.Production),
|
||||
description = Some(
|
||||
PlainText("The User SimClusters Embedding Endpoint in Representation Management Service." +
|
||||
" TDD: http://go/rms-tdd"))
|
||||
)
|
||||
|
||||
override def fetch(key: Key, view: View): Stitch[Result[Value]] = {
|
||||
val embeddingId = SimClustersEmbeddingId(
|
||||
view.embeddingType,
|
||||
view.modelVersion,
|
||||
InternalId.UserId(key)
|
||||
)
|
||||
|
||||
storeStitch(embeddingId)
|
||||
.map(embedding => found(embedding))
|
||||
.handle {
|
||||
case stitch.NotFound => missing
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"decider/src/main/scala",
|
||||
"finagle/finagle-memcached",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
|
||||
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection",
|
||||
"src/scala/com/twitter/simclusters_v2/common",
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
package com.twitter.representation_manager.common
|
||||
|
||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.hashing.KeyHasher
|
||||
import com.twitter.hermit.store.common.ObservedMemcachedReadableStore
|
||||
import com.twitter.relevance_platform.common.injection.LZ4Injection
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbeddingIdCacheKeyBuilder
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType._
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion._
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.util.Duration
|
||||
|
||||
/*
|
||||
* NOTE - ALL the cache configs here are just placeholders, NONE of them is used anyweher in RMS yet
|
||||
* */
|
||||
sealed trait MemCacheParams
|
||||
sealed trait MemCacheConfig
|
||||
|
||||
/*
|
||||
* This holds params that is required to set up a memcache cache for a single embedding store
|
||||
* */
|
||||
case class EnabledMemCacheParams(ttl: Duration) extends MemCacheParams
|
||||
object DisabledMemCacheParams extends MemCacheParams
|
||||
|
||||
/*
|
||||
* We use this MemcacheConfig as the single source to set up the memcache for all RMS use cases
|
||||
* NO OVERRIDE FROM CLIENT
|
||||
* */
|
||||
object MemCacheConfig {
|
||||
val keyHasher: KeyHasher = KeyHasher.FNV1A_64
|
||||
val hashKeyPrefix: String = "RMS"
|
||||
val simclustersEmbeddingCacheKeyBuilder =
|
||||
SimClustersEmbeddingIdCacheKeyBuilder(keyHasher.hashKey, hashKeyPrefix)
|
||||
|
||||
val cacheParamsMap: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
MemCacheParams
|
||||
] = Map(
|
||||
// Tweet Embeddings
|
||||
(LogFavBasedTweet, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 10.minutes),
|
||||
(LogFavBasedTweet, Model20m145k2020) -> EnabledMemCacheParams(ttl = 10.minutes),
|
||||
(LogFavLongestL2EmbeddingTweet, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 10.minutes),
|
||||
(LogFavLongestL2EmbeddingTweet, Model20m145k2020) -> EnabledMemCacheParams(ttl = 10.minutes),
|
||||
// User - KnownFor Embeddings
|
||||
(FavBasedProducer, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FollowBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(AggregatableLogFavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(RelaxedAggregatableLogFavBasedProducer, Model20m145kUpdated) -> EnabledMemCacheParams(ttl =
|
||||
12.hours),
|
||||
(RelaxedAggregatableLogFavBasedProducer, Model20m145k2020) -> EnabledMemCacheParams(ttl =
|
||||
12.hours),
|
||||
// User - InterestedIn Embeddings
|
||||
(LogFavBasedUserInterestedInFromAPE, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FollowBasedUserInterestedInFromAPE, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FavBasedUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FavBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FollowBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(LogFavBasedUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FavBasedUserInterestedInFromPE, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FilteredUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FilteredUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(FilteredUserInterestedInFromPE, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(UnfilteredUserInterestedIn, Model20m145kUpdated) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(UnfilteredUserInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(UserNextInterestedIn, Model20m145k2020) -> EnabledMemCacheParams(ttl =
|
||||
30.minutes), //embedding is updated every 2 hours, keeping it lower to avoid staleness
|
||||
(
|
||||
LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(
|
||||
LogFavBasedUserInterestedAverageAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(
|
||||
LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(
|
||||
LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(
|
||||
LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(
|
||||
LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
// Topic Embeddings
|
||||
(FavTfgTopic, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
(LogFavBasedKgoApeTopic, Model20m145k2020) -> EnabledMemCacheParams(ttl = 12.hours),
|
||||
)
|
||||
|
||||
def getCacheSetup(
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): MemCacheParams = {
|
||||
// When requested (embeddingType, modelVersion) doesn't exist, we return DisabledMemCacheParams
|
||||
cacheParamsMap.getOrElse((embeddingType, modelVersion), DisabledMemCacheParams)
|
||||
}
|
||||
|
||||
def getCacheKeyPrefix(embeddingType: EmbeddingType, modelVersion: ModelVersion) =
|
||||
s"${embeddingType.value}_${modelVersion.value}_"
|
||||
|
||||
def getStatsName(embeddingType: EmbeddingType, modelVersion: ModelVersion) =
|
||||
s"${embeddingType.name}_${modelVersion.name}_mem_cache"
|
||||
|
||||
/**
|
||||
* Build a ReadableStore based on MemCacheConfig.
|
||||
*
|
||||
* If memcache is disabled, it will return a normal readable store wrapper of the rawStore,
|
||||
* with SimClustersEmbedding as value;
|
||||
* If memcache is enabled, it will return a ObservedMemcachedReadableStore wrapper of the rawStore,
|
||||
* with memcache set up according to the EnabledMemCacheParams
|
||||
* */
|
||||
def buildMemCacheStoreForSimClustersEmbedding(
|
||||
rawStore: ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding],
|
||||
cacheClient: Client,
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion,
|
||||
stats: StatsReceiver
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val cacheParams = getCacheSetup(embeddingType, modelVersion)
|
||||
val store = cacheParams match {
|
||||
case DisabledMemCacheParams => rawStore
|
||||
case EnabledMemCacheParams(ttl) =>
|
||||
val memCacheKeyPrefix = MemCacheConfig.getCacheKeyPrefix(
|
||||
embeddingType,
|
||||
modelVersion
|
||||
)
|
||||
val statsName = MemCacheConfig.getStatsName(
|
||||
embeddingType,
|
||||
modelVersion
|
||||
)
|
||||
ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = rawStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = ttl
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = stats.scope(statsName),
|
||||
keyToString = { k => memCacheKeyPrefix + k.toString }
|
||||
)
|
||||
}
|
||||
store.mapValues(SimClustersEmbedding(_))
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
package com.twitter.representation_manager.common
|
||||
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.decider.RandomRecipient
|
||||
import com.twitter.decider.Recipient
|
||||
import com.twitter.simclusters_v2.common.DeciderGateBuilderWithIdHashing
|
||||
import javax.inject.Inject
|
||||
|
||||
case class RepresentationManagerDecider @Inject() (decider: Decider) {
|
||||
|
||||
val deciderGateBuilder = new DeciderGateBuilderWithIdHashing(decider)
|
||||
|
||||
def isAvailable(feature: String, recipient: Option[Recipient]): Boolean = {
|
||||
decider.isAvailable(feature, recipient)
|
||||
}
|
||||
|
||||
/**
|
||||
* When useRandomRecipient is set to false, the decider is either completely on or off.
|
||||
* When useRandomRecipient is set to true, the decider is on for the specified % of traffic.
|
||||
*/
|
||||
def isAvailable(feature: String, useRandomRecipient: Boolean = true): Boolean = {
|
||||
if (useRandomRecipient) isAvailable(feature, Some(RandomRecipient))
|
||||
else isAvailable(feature, None)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"content-recommender/server/src/main/scala/com/twitter/contentrecommender:representation-manager-deps",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/store/strato",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
|
||||
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/injection",
|
||||
"relevance-platform/src/main/scala/com/twitter/relevance_platform/common/readablestore",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/common",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/store",
|
||||
"src/scala/com/twitter/ml/api/embedding",
|
||||
"src/scala/com/twitter/simclusters_v2/common",
|
||||
"src/scala/com/twitter/simclusters_v2/score",
|
||||
"src/scala/com/twitter/simclusters_v2/summingbird/stores",
|
||||
"src/scala/com/twitter/storehaus_internal/manhattan",
|
||||
"src/scala/com/twitter/storehaus_internal/util",
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
|
||||
"src/thrift/com/twitter/socialgraph:thrift-scala",
|
||||
"storage/clients/manhattan/client/src/main/scala",
|
||||
"tweetypie/src/scala/com/twitter/tweetypie/util",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,846 @@
|
|||
package com.twitter.representation_manager.migration
|
||||
|
||||
import com.twitter.bijection.Injection
|
||||
import com.twitter.bijection.scrooge.BinaryScalaCodec
|
||||
import com.twitter.contentrecommender.store.ApeEntityEmbeddingStore
|
||||
import com.twitter.contentrecommender.store.InterestsOptOutStore
|
||||
import com.twitter.contentrecommender.store.SemanticCoreTopicSeedStore
|
||||
import com.twitter.contentrecommender.twistly
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.escherbird.util.uttclient.CacheConfigV2
|
||||
import com.twitter.escherbird.util.uttclient.CachedUttClientV2
|
||||
import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2
|
||||
import com.twitter.escherbird.utt.strato.thriftscala.Environment
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax
|
||||
import com.twitter.finagle.mux.ClientDiscardedRequestException
|
||||
import com.twitter.finagle.service.ReqRep
|
||||
import com.twitter.finagle.service.ResponseClass
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.frigate.common.store.strato.StratoFetchableStore
|
||||
import com.twitter.frigate.common.util.SeqLongInjection
|
||||
import com.twitter.hashing.KeyHasher
|
||||
import com.twitter.hermit.store.common.DeciderableReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedCachedReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedMemcachedReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedReadableStore
|
||||
import com.twitter.interests.thriftscala.InterestsThriftService
|
||||
import com.twitter.relevance_platform.common.injection.LZ4Injection
|
||||
import com.twitter.relevance_platform.common.readablestore.ReadableStoreWithTimeout
|
||||
import com.twitter.representation_manager.common.RepresentationManagerDecider
|
||||
import com.twitter.representation_manager.store.DeciderConstants
|
||||
import com.twitter.representation_manager.store.DeciderKey
|
||||
import com.twitter.simclusters_v2.common.ModelVersions
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbeddingIdCacheKeyBuilder
|
||||
import com.twitter.simclusters_v2.stores.SimClustersEmbeddingStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.PersistentTweetEmbeddingStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.ProducerClusterEmbeddingReadableStores
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore
|
||||
import com.twitter.simclusters_v2.thriftscala.ClustersUserIsInterestedIn
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType._
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion.Model20m145k2020
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion.Model20m145kUpdated
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersMultiEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersMultiEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.storehaus_internal.manhattan.Athena
|
||||
import com.twitter.storehaus_internal.manhattan.ManhattanRO
|
||||
import com.twitter.storehaus_internal.manhattan.ManhattanROConfig
|
||||
import com.twitter.storehaus_internal.util.ApplicationID
|
||||
import com.twitter.storehaus_internal.util.DatasetName
|
||||
import com.twitter.storehaus_internal.util.HDFSPath
|
||||
import com.twitter.strato.client.Strato
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
import com.twitter.tweetypie.util.UserId
|
||||
import com.twitter.util.Duration
|
||||
import com.twitter.util.Future
|
||||
import com.twitter.util.Throw
|
||||
import com.twitter.util.Timer
|
||||
import javax.inject.Inject
|
||||
import javax.inject.Named
|
||||
import scala.reflect.ClassTag
|
||||
|
||||
class LegacyRMS @Inject() (
|
||||
serviceIdentifier: ServiceIdentifier,
|
||||
cacheClient: Client,
|
||||
stats: StatsReceiver,
|
||||
decider: Decider,
|
||||
clientId: ClientId,
|
||||
timer: Timer,
|
||||
@Named("cacheHashKeyPrefix") val cacheHashKeyPrefix: String = "RMS",
|
||||
@Named("useContentRecommenderConfiguration") val useContentRecommenderConfiguration: Boolean =
|
||||
false) {
|
||||
|
||||
private val mhMtlsParams: ManhattanKVClientMtlsParams = ManhattanKVClientMtlsParams(
|
||||
serviceIdentifier)
|
||||
private val rmsDecider = RepresentationManagerDecider(decider)
|
||||
val keyHasher: KeyHasher = KeyHasher.FNV1A_64
|
||||
|
||||
private val embeddingCacheKeyBuilder =
|
||||
SimClustersEmbeddingIdCacheKeyBuilder(keyHasher.hashKey, cacheHashKeyPrefix)
|
||||
private val statsReceiver = stats.scope("representation_management")
|
||||
|
||||
// Strato client, default timeout = 280ms
|
||||
val stratoClient: StratoClient =
|
||||
Strato.client
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.build()
|
||||
|
||||
// Builds ThriftMux client builder for Content-Recommender service
|
||||
private def makeThriftClientBuilder(
|
||||
requestTimeout: Duration
|
||||
): ThriftMux.Client = {
|
||||
ThriftMux.client
|
||||
.withClientId(clientId)
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.withRequestTimeout(requestTimeout)
|
||||
.withStatsReceiver(statsReceiver.scope("clnt"))
|
||||
.withResponseClassifier {
|
||||
case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable
|
||||
}
|
||||
}
|
||||
|
||||
private def makeThriftClient[ThriftServiceType: ClassTag](
|
||||
dest: String,
|
||||
label: String,
|
||||
requestTimeout: Duration = 450.milliseconds
|
||||
): ThriftServiceType = {
|
||||
makeThriftClientBuilder(requestTimeout)
|
||||
.build[ThriftServiceType](dest, label)
|
||||
}
|
||||
|
||||
/** *** SimCluster Embedding Stores ******/
|
||||
implicit val simClustersEmbeddingIdInjection: Injection[SimClustersEmbeddingId, Array[Byte]] =
|
||||
BinaryScalaCodec(SimClustersEmbeddingId)
|
||||
implicit val simClustersEmbeddingInjection: Injection[ThriftSimClustersEmbedding, Array[Byte]] =
|
||||
BinaryScalaCodec(ThriftSimClustersEmbedding)
|
||||
implicit val simClustersMultiEmbeddingInjection: Injection[SimClustersMultiEmbedding, Array[
|
||||
Byte
|
||||
]] =
|
||||
BinaryScalaCodec(SimClustersMultiEmbedding)
|
||||
implicit val simClustersMultiEmbeddingIdInjection: Injection[SimClustersMultiEmbeddingId, Array[
|
||||
Byte
|
||||
]] =
|
||||
BinaryScalaCodec(SimClustersMultiEmbeddingId)
|
||||
|
||||
def getEmbeddingsDataset(
|
||||
mhMtlsParams: ManhattanKVClientMtlsParams,
|
||||
datasetName: String
|
||||
): ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding] = {
|
||||
ManhattanRO.getReadableStoreWithMtls[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
ManhattanROConfig(
|
||||
HDFSPath(""), // not needed
|
||||
ApplicationID("content_recommender_athena"),
|
||||
DatasetName(datasetName), // this should be correct
|
||||
Athena
|
||||
),
|
||||
mhMtlsParams
|
||||
)
|
||||
}
|
||||
|
||||
lazy val logFavBasedLongestL2Tweet20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.longestL2NormTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset,
|
||||
statsReceiver,
|
||||
maxLength = 10,
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = rawStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 15.minutes
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver =
|
||||
statsReceiver.scope("log_fav_based_longest_l2_tweet_embedding_20m145k2020_mem_cache"),
|
||||
keyToString = { k =>
|
||||
s"scez_l2:${LogFavBasedTweet}_${ModelVersions.Model20M145K2020}_$k"
|
||||
}
|
||||
)
|
||||
|
||||
val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] =
|
||||
memcachedStore
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
LogFavLongestL2EmbeddingTweet,
|
||||
Model20m145k2020,
|
||||
InternalId.TweetId(tweetId)) =>
|
||||
tweetId
|
||||
}
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
inMemoryCacheStore,
|
||||
ttl = 12.minute,
|
||||
maxKeys = 1048575,
|
||||
cacheName = "log_fav_based_longest_l2_tweet_embedding_20m145k2020_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_longest_l2_tweet_embedding_20m145k2020_store"))
|
||||
}
|
||||
|
||||
lazy val logFavBased20M145KUpdatedTweetEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.mostRecentTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145kUpdatedDataset,
|
||||
statsReceiver
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = rawStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 10.minutes
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_tweet_embedding_mem_cache"),
|
||||
keyToString = { k =>
|
||||
// SimClusters_embedding_LZ4/embeddingType_modelVersion_tweetId
|
||||
s"scez:${LogFavBasedTweet}_${ModelVersions.Model20M145KUpdated}_$k"
|
||||
}
|
||||
)
|
||||
|
||||
val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
memcachedStore
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
LogFavBasedTweet,
|
||||
Model20m145kUpdated,
|
||||
InternalId.TweetId(tweetId)) =>
|
||||
tweetId
|
||||
}
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
}
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
inMemoryCacheStore,
|
||||
ttl = 5.minute,
|
||||
maxKeys = 1048575, // 200MB
|
||||
cacheName = "log_fav_based_tweet_embedding_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_tweet_embedding_store"))
|
||||
}
|
||||
|
||||
lazy val logFavBased20M145K2020TweetEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.mostRecentTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset,
|
||||
statsReceiver,
|
||||
maxLength = 10,
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = rawStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 15.minutes
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_tweet_embedding_20m145k2020_mem_cache"),
|
||||
keyToString = { k =>
|
||||
// SimClusters_embedding_LZ4/embeddingType_modelVersion_tweetId
|
||||
s"scez:${LogFavBasedTweet}_${ModelVersions.Model20M145K2020}_$k"
|
||||
}
|
||||
)
|
||||
|
||||
val inMemoryCacheStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] =
|
||||
memcachedStore
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
LogFavBasedTweet,
|
||||
Model20m145k2020,
|
||||
InternalId.TweetId(tweetId)) =>
|
||||
tweetId
|
||||
}
|
||||
.mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
inMemoryCacheStore,
|
||||
ttl = 12.minute,
|
||||
maxKeys = 16777215,
|
||||
cacheName = "log_fav_based_tweet_embedding_20m145k2020_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_tweet_embedding_20m145k2020_store"))
|
||||
}
|
||||
|
||||
lazy val favBasedTfgTopicEmbedding2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val stratoStore =
|
||||
StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/favBasedTFGTopic20M145K2020")
|
||||
|
||||
val truncatedStore = stratoStore.mapValues { embedding =>
|
||||
SimClustersEmbedding(embedding, truncate = 50)
|
||||
}
|
||||
|
||||
ObservedCachedReadableStore.from(
|
||||
ObservedReadableStore(truncatedStore)(
|
||||
statsReceiver.scope("fav_tfg_topic_embedding_2020_cache_backing_store")),
|
||||
ttl = 12.hours,
|
||||
maxKeys = 262143, // 200MB
|
||||
cacheName = "fav_tfg_topic_embedding_2020_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("fav_tfg_topic_embedding_2020_cache"))
|
||||
}
|
||||
|
||||
lazy val logFavBasedApe20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
ObservedReadableStore(
|
||||
StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/logFavBasedAPE20M145K2020")
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
AggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
internalId) =>
|
||||
SimClustersEmbeddingId(AggregatableLogFavBasedProducer, Model20m145k2020, internalId)
|
||||
}
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding, 50))
|
||||
)(statsReceiver.scope("aggregatable_producer_embeddings_by_logfav_score_2020"))
|
||||
}
|
||||
|
||||
val interestService: InterestsThriftService.MethodPerEndpoint =
|
||||
makeThriftClient[InterestsThriftService.MethodPerEndpoint](
|
||||
"/s/interests-thrift-service/interests-thrift-service",
|
||||
"interests_thrift_service"
|
||||
)
|
||||
|
||||
val interestsOptOutStore: InterestsOptOutStore = InterestsOptOutStore(interestService)
|
||||
|
||||
// Save 2 ^ 18 UTTs. Promising 100% cache rate
|
||||
lazy val defaultCacheConfigV2: CacheConfigV2 = CacheConfigV2(262143)
|
||||
lazy val uttClientCacheConfigsV2: UttClientCacheConfigsV2 = UttClientCacheConfigsV2(
|
||||
getTaxonomyConfig = defaultCacheConfigV2,
|
||||
getUttTaxonomyConfig = defaultCacheConfigV2,
|
||||
getLeafIds = defaultCacheConfigV2,
|
||||
getLeafUttEntities = defaultCacheConfigV2
|
||||
)
|
||||
|
||||
// CachedUttClient to use StratoClient
|
||||
lazy val cachedUttClientV2: CachedUttClientV2 = new CachedUttClientV2(
|
||||
stratoClient = stratoClient,
|
||||
env = Environment.Prod,
|
||||
cacheConfigs = uttClientCacheConfigsV2,
|
||||
statsReceiver = statsReceiver.scope("cached_utt_client")
|
||||
)
|
||||
|
||||
lazy val semanticCoreTopicSeedStore: ReadableStore[
|
||||
SemanticCoreTopicSeedStore.Key,
|
||||
Seq[UserId]
|
||||
] = {
|
||||
/*
|
||||
Up to 1000 Long seeds per topic/language = 62.5kb per topic/language (worst case)
|
||||
Assume ~10k active topic/languages ~= 650MB (worst case)
|
||||
*/
|
||||
val underlying = new SemanticCoreTopicSeedStore(cachedUttClientV2, interestsOptOutStore)(
|
||||
statsReceiver.scope("semantic_core_topic_seed_store"))
|
||||
|
||||
val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = underlying,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = SeqLongInjection,
|
||||
statsReceiver = statsReceiver.scope("topic_producer_seed_store_mem_cache"),
|
||||
keyToString = { k => s"tpss:${k.entityId}_${k.languageCode}" }
|
||||
)
|
||||
|
||||
ObservedCachedReadableStore.from[SemanticCoreTopicSeedStore.Key, Seq[UserId]](
|
||||
store = memcacheStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 20e3.toInt,
|
||||
cacheName = "topic_producer_seed_store_cache",
|
||||
windowSize = 5000
|
||||
)(statsReceiver.scope("topic_producer_seed_store_cache"))
|
||||
}
|
||||
|
||||
lazy val logFavBasedApeEntity20M145K2020EmbeddingStore: ApeEntityEmbeddingStore = {
|
||||
val apeStore = logFavBasedApe20M145K2020EmbeddingStore.composeKeyMapping[UserId]({ id =>
|
||||
SimClustersEmbeddingId(
|
||||
AggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
InternalId.UserId(id))
|
||||
})
|
||||
|
||||
new ApeEntityEmbeddingStore(
|
||||
semanticCoreSeedStore = semanticCoreTopicSeedStore,
|
||||
aggregatableProducerEmbeddingStore = apeStore,
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_store"))
|
||||
}
|
||||
|
||||
lazy val logFavBasedApeEntity20M145K2020EmbeddingCachedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val truncatedStore =
|
||||
logFavBasedApeEntity20M145K2020EmbeddingStore.mapValues(_.truncate(50).toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = truncatedStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
val inMemoryCachedStore =
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "log_fav_based_ape_entity_2020_embedding_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_cached_store"))
|
||||
|
||||
DeciderableReadableStore(
|
||||
inMemoryCachedStore,
|
||||
rmsDecider.deciderGateBuilder.idGateWithHashing[SimClustersEmbeddingId](
|
||||
DeciderKey.enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore),
|
||||
statsReceiver.scope("log_fav_based_ape_entity_2020_embedding_deciderable_store")
|
||||
)
|
||||
}
|
||||
|
||||
lazy val relaxedLogFavBasedApe20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
ObservedReadableStore(
|
||||
StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/logFavBasedAPERelaxedFavEngagementThreshold20M145K2020")
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
internalId) =>
|
||||
SimClustersEmbeddingId(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
internalId)
|
||||
}
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding).truncate(50))
|
||||
)(statsReceiver.scope(
|
||||
"aggregatable_producer_embeddings_by_logfav_score_relaxed_fav_engagement_threshold_2020"))
|
||||
}
|
||||
|
||||
lazy val relaxedLogFavBasedApe20M145K2020EmbeddingCachedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val truncatedStore =
|
||||
relaxedLogFavBasedApe20M145K2020EmbeddingStore.mapValues(_.truncate(50).toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = truncatedStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver =
|
||||
statsReceiver.scope("relaxed_log_fav_based_ape_entity_2020_embedding_mem_cache"),
|
||||
keyToString = { k: SimClustersEmbeddingId => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "relaxed_log_fav_based_ape_entity_2020_embedding_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("relaxed_log_fav_based_ape_entity_2020_embedding_cache_store"))
|
||||
}
|
||||
|
||||
lazy val favBasedProducer20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore = ProducerClusterEmbeddingReadableStores
|
||||
.getProducerTopKSimClusters2020EmbeddingsStore(
|
||||
mhMtlsParams
|
||||
).composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
FavBasedProducer,
|
||||
Model20m145k2020,
|
||||
InternalId.UserId(userId)) =>
|
||||
userId
|
||||
}.mapValues { topSimClustersWithScore =>
|
||||
ThriftSimClustersEmbedding(topSimClustersWithScore.topClusters.take(10))
|
||||
}
|
||||
|
||||
// same memcache config as for favBasedUserInterestedIn20M145K2020Store
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 24.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("fav_based_producer_embedding_20M_145K_2020_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 12.hours,
|
||||
maxKeys = 16777215,
|
||||
cacheName = "fav_based_producer_embedding_20M_145K_2020_embedding_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("fav_based_producer_embedding_20M_145K_2020_embedding_store"))
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val interestedIn20M145KUpdatedStore: ReadableStore[UserId, ClustersUserIsInterestedIn] = {
|
||||
UserInterestedInReadableStore.defaultStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145KUpdated
|
||||
)
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val interestedIn20M145K2020Store: ReadableStore[UserId, ClustersUserIsInterestedIn] = {
|
||||
UserInterestedInReadableStore.defaultStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145K2020
|
||||
)
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val InterestedInFromPE20M145KUpdatedStore: ReadableStore[
|
||||
UserId,
|
||||
ClustersUserIsInterestedIn
|
||||
] = {
|
||||
UserInterestedInReadableStore.defaultIIPEStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145KUpdated)
|
||||
}
|
||||
|
||||
lazy val simClustersInterestedInStore: ReadableStore[
|
||||
(UserId, ModelVersion),
|
||||
ClustersUserIsInterestedIn
|
||||
] = {
|
||||
new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] {
|
||||
override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = {
|
||||
k match {
|
||||
case (userId, Model20m145kUpdated) =>
|
||||
interestedIn20M145KUpdatedStore.get(userId)
|
||||
case (userId, Model20m145k2020) =>
|
||||
interestedIn20M145K2020Store.get(userId)
|
||||
case _ =>
|
||||
Future.None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lazy val simClustersInterestedInFromProducerEmbeddingsStore: ReadableStore[
|
||||
(UserId, ModelVersion),
|
||||
ClustersUserIsInterestedIn
|
||||
] = {
|
||||
new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] {
|
||||
override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = {
|
||||
k match {
|
||||
case (userId, ModelVersion.Model20m145kUpdated) =>
|
||||
InterestedInFromPE20M145KUpdatedStore.get(userId)
|
||||
case _ =>
|
||||
Future.None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
lazy val userInterestedInStore =
|
||||
new twistly.interestedin.EmbeddingStore(
|
||||
interestedInStore = simClustersInterestedInStore,
|
||||
interestedInFromProducerEmbeddingStore = simClustersInterestedInFromProducerEmbeddingsStore,
|
||||
statsReceiver = statsReceiver
|
||||
)
|
||||
|
||||
// Production
|
||||
lazy val favBasedUserInterestedIn20M145KUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore =
|
||||
UserInterestedInReadableStore
|
||||
.defaultSimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.FavBasedUserInterestedIn,
|
||||
ModelVersion.Model20m145kUpdated)
|
||||
.mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("fav_based_user_interested_in_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "fav_based_user_interested_in_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("fav_based_user_interested_in_store"))
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val LogFavBasedInterestedInFromAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore =
|
||||
UserInterestedInReadableStore
|
||||
.defaultIIAPESimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.LogFavBasedUserInterestedInFromAPE,
|
||||
ModelVersion.Model20m145k2020)
|
||||
.mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_user_interested_in_from_ape_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "log_fav_based_user_interested_in_from_ape_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_user_interested_in_from_ape_store"))
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val FollowBasedInterestedInFromAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore =
|
||||
UserInterestedInReadableStore
|
||||
.defaultIIAPESimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.FollowBasedUserInterestedInFromAPE,
|
||||
ModelVersion.Model20m145k2020)
|
||||
.mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("follow_based_user_interested_in_from_ape_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "follow_based_user_interested_in_from_ape_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("follow_based_user_interested_in_from_ape_store"))
|
||||
}
|
||||
|
||||
// production
|
||||
lazy val favBasedUserInterestedIn20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore: ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding] =
|
||||
UserInterestedInReadableStore
|
||||
.defaultSimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.FavBasedUserInterestedIn,
|
||||
ModelVersion.Model20m145k2020).mapValues(_.toThrift)
|
||||
|
||||
ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("fav_based_user_interested_in_2020_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val logFavBasedUserInterestedIn20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore =
|
||||
UserInterestedInReadableStore
|
||||
.defaultSimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.LogFavBasedUserInterestedIn,
|
||||
ModelVersion.Model20m145k2020)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore.mapValues(_.toThrift),
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("log_fav_based_user_interested_in_2020_store"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "log_fav_based_user_interested_in_2020_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("log_fav_based_user_interested_in_2020_store"))
|
||||
}
|
||||
|
||||
// Production
|
||||
lazy val favBasedUserInterestedInFromPE20M145KUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val underlyingStore =
|
||||
UserInterestedInReadableStore
|
||||
.defaultIIPESimClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
EmbeddingType.FavBasedUserInterestedInFromPE,
|
||||
ModelVersion.Model20m145kUpdated)
|
||||
.mapValues(_.toThrift)
|
||||
|
||||
val memcachedStore = ObservedMemcachedReadableStore
|
||||
.fromCacheClient(
|
||||
backingStore = underlyingStore,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours
|
||||
)(
|
||||
valueInjection = LZ4Injection.compose(BinaryScalaCodec(ThriftSimClustersEmbedding)),
|
||||
statsReceiver = statsReceiver.scope("fav_based_user_interested_in_from_pe_mem_cache"),
|
||||
keyToString = { k => embeddingCacheKeyBuilder.apply(k) }
|
||||
).mapValues(SimClustersEmbedding(_))
|
||||
|
||||
ObservedCachedReadableStore.from[SimClustersEmbeddingId, SimClustersEmbedding](
|
||||
memcachedStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 262143,
|
||||
cacheName = "fav_based_user_interested_in_from_pe_cache",
|
||||
windowSize = 10000L
|
||||
)(statsReceiver.scope("fav_based_user_interested_in_from_pe_cache"))
|
||||
}
|
||||
|
||||
private val underlyingStores: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding]
|
||||
] = Map(
|
||||
// Tweet Embeddings
|
||||
(LogFavBasedTweet, Model20m145kUpdated) -> logFavBased20M145KUpdatedTweetEmbeddingStore,
|
||||
(LogFavBasedTweet, Model20m145k2020) -> logFavBased20M145K2020TweetEmbeddingStore,
|
||||
(
|
||||
LogFavLongestL2EmbeddingTweet,
|
||||
Model20m145k2020) -> logFavBasedLongestL2Tweet20M145K2020EmbeddingStore,
|
||||
// Entity Embeddings
|
||||
(FavTfgTopic, Model20m145k2020) -> favBasedTfgTopicEmbedding2020Store,
|
||||
(
|
||||
LogFavBasedKgoApeTopic,
|
||||
Model20m145k2020) -> logFavBasedApeEntity20M145K2020EmbeddingCachedStore,
|
||||
// KnownFor Embeddings
|
||||
(FavBasedProducer, Model20m145k2020) -> favBasedProducer20M145K2020EmbeddingStore,
|
||||
(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020) -> relaxedLogFavBasedApe20M145K2020EmbeddingCachedStore,
|
||||
// InterestedIn Embeddings
|
||||
(
|
||||
LogFavBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020) -> LogFavBasedInterestedInFromAPE20M145K2020Store,
|
||||
(
|
||||
FollowBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020) -> FollowBasedInterestedInFromAPE20M145K2020Store,
|
||||
(FavBasedUserInterestedIn, Model20m145kUpdated) -> favBasedUserInterestedIn20M145KUpdatedStore,
|
||||
(FavBasedUserInterestedIn, Model20m145k2020) -> favBasedUserInterestedIn20M145K2020Store,
|
||||
(LogFavBasedUserInterestedIn, Model20m145k2020) -> logFavBasedUserInterestedIn20M145K2020Store,
|
||||
(
|
||||
FavBasedUserInterestedInFromPE,
|
||||
Model20m145kUpdated) -> favBasedUserInterestedInFromPE20M145KUpdatedStore,
|
||||
(FilteredUserInterestedIn, Model20m145kUpdated) -> userInterestedInStore,
|
||||
(FilteredUserInterestedIn, Model20m145k2020) -> userInterestedInStore,
|
||||
(FilteredUserInterestedInFromPE, Model20m145kUpdated) -> userInterestedInStore,
|
||||
(UnfilteredUserInterestedIn, Model20m145kUpdated) -> userInterestedInStore,
|
||||
(UnfilteredUserInterestedIn, Model20m145k2020) -> userInterestedInStore,
|
||||
)
|
||||
|
||||
val simClustersEmbeddingStore: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val underlying: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] =
|
||||
SimClustersEmbeddingStore.buildWithDecider(
|
||||
underlyingStores = underlyingStores,
|
||||
decider = rmsDecider.decider,
|
||||
statsReceiver = statsReceiver.scope("simClusters_embeddings_store_deciderable")
|
||||
)
|
||||
|
||||
val underlyingWithTimeout: ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] =
|
||||
new ReadableStoreWithTimeout(
|
||||
rs = underlying,
|
||||
decider = rmsDecider.decider,
|
||||
enableTimeoutDeciderKey = DeciderConstants.enableSimClustersEmbeddingStoreTimeouts,
|
||||
timeoutValueKey = DeciderConstants.simClustersEmbeddingStoreTimeoutValueMillis,
|
||||
timer = timer,
|
||||
statsReceiver = statsReceiver.scope("simClusters_embedding_store_timeouts")
|
||||
)
|
||||
|
||||
ObservedReadableStore(
|
||||
store = underlyingWithTimeout
|
||||
)(statsReceiver.scope("simClusters_embeddings_store"))
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"finagle-internal/mtls/src/main/scala/com/twitter/finagle/mtls/authentication",
|
||||
"finagle/finagle-stats",
|
||||
"finatra/inject/inject-core/src/main/scala",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util",
|
||||
"interests-service/thrift/src/main/thrift:thrift-scala",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/common",
|
||||
"servo/util",
|
||||
"src/scala/com/twitter/storehaus_internal/manhattan",
|
||||
"src/scala/com/twitter/storehaus_internal/memcache",
|
||||
"src/scala/com/twitter/storehaus_internal/util",
|
||||
"strato/src/main/scala/com/twitter/strato/client",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import javax.inject.Singleton
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.storehaus_internal.memcache.MemcacheStore
|
||||
import com.twitter.storehaus_internal.util.ClientName
|
||||
import com.twitter.storehaus_internal.util.ZkEndPoint
|
||||
|
||||
object CacheModule extends TwitterModule {
|
||||
|
||||
private val cacheDest = flag[String]("cache_module.dest", "Path to memcache service")
|
||||
private val timeout = flag[Int]("memcache.timeout", "Memcache client timeout")
|
||||
private val retries = flag[Int]("memcache.retries", "Memcache timeout retries")
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesCache(
|
||||
serviceIdentifier: ServiceIdentifier,
|
||||
stats: StatsReceiver
|
||||
): Client =
|
||||
MemcacheStore.memcachedClient(
|
||||
name = ClientName("memcache_representation_manager"),
|
||||
dest = ZkEndPoint(cacheDest()),
|
||||
timeout = timeout().milliseconds,
|
||||
retries = retries(),
|
||||
statsReceiver = stats.scope("cache_client"),
|
||||
serviceIdentifier = serviceIdentifier
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.finagle.ThriftMux
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.finagle.mtls.client.MtlsStackClient.MtlsThriftMuxClientSyntax
|
||||
import com.twitter.finagle.mux.ClientDiscardedRequestException
|
||||
import com.twitter.finagle.service.ReqRep
|
||||
import com.twitter.finagle.service.ResponseClass
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.finagle.thrift.ClientId
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.interests.thriftscala.InterestsThriftService
|
||||
import com.twitter.util.Throw
|
||||
import javax.inject.Singleton
|
||||
|
||||
object InterestsThriftClientModule extends TwitterModule {
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesInterestsThriftClient(
|
||||
clientId: ClientId,
|
||||
serviceIdentifier: ServiceIdentifier,
|
||||
statsReceiver: StatsReceiver
|
||||
): InterestsThriftService.MethodPerEndpoint = {
|
||||
ThriftMux.client
|
||||
.withClientId(clientId)
|
||||
.withMutualTls(serviceIdentifier)
|
||||
.withRequestTimeout(450.milliseconds)
|
||||
.withStatsReceiver(statsReceiver.scope("InterestsThriftClient"))
|
||||
.withResponseClassifier {
|
||||
case ReqRep(_, Throw(_: ClientDiscardedRequestException)) => ResponseClass.Ignorable
|
||||
}
|
||||
.build[InterestsThriftService.MethodPerEndpoint](
|
||||
dest = "/s/interests-thrift-service/interests-thrift-service",
|
||||
label = "interests_thrift_service"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.inject.TwitterModule
|
||||
import javax.inject.Named
|
||||
import javax.inject.Singleton
|
||||
|
||||
object LegacyRMSConfigModule extends TwitterModule {
|
||||
@Singleton
|
||||
@Provides
|
||||
@Named("cacheHashKeyPrefix")
|
||||
def providesCacheHashKeyPrefix: String = "RMS"
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
@Named("useContentRecommenderConfiguration")
|
||||
def providesUseContentRecommenderConfiguration: Boolean = false
|
||||
}
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import javax.inject.Singleton
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.decider.Decider
|
||||
import com.twitter.finagle.mtls.authentication.ServiceIdentifier
|
||||
import com.twitter.representation_manager.common.RepresentationManagerDecider
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
|
||||
object StoreModule extends TwitterModule {
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesMhMtlsParams(
|
||||
serviceIdentifier: ServiceIdentifier
|
||||
): ManhattanKVClientMtlsParams = ManhattanKVClientMtlsParams(serviceIdentifier)
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesRmsDecider(
|
||||
decider: Decider
|
||||
): RepresentationManagerDecider = RepresentationManagerDecider(decider)
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.finagle.util.DefaultTimer
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.util.Timer
|
||||
import javax.inject.Singleton
|
||||
|
||||
object TimerModule extends TwitterModule {
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesTimer: Timer = DefaultTimer
|
||||
}
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package com.twitter.representation_manager.modules
|
||||
|
||||
import com.google.inject.Provides
|
||||
import com.twitter.escherbird.util.uttclient.CacheConfigV2
|
||||
import com.twitter.escherbird.util.uttclient.CachedUttClientV2
|
||||
import com.twitter.escherbird.util.uttclient.UttClientCacheConfigsV2
|
||||
import com.twitter.escherbird.utt.strato.thriftscala.Environment
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.inject.TwitterModule
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import javax.inject.Singleton
|
||||
|
||||
object UttClientModule extends TwitterModule {
|
||||
|
||||
@Singleton
|
||||
@Provides
|
||||
def providesUttClient(
|
||||
stratoClient: StratoClient,
|
||||
statsReceiver: StatsReceiver
|
||||
): CachedUttClientV2 = {
|
||||
// Save 2 ^ 18 UTTs. Promising 100% cache rate
|
||||
val defaultCacheConfigV2: CacheConfigV2 = CacheConfigV2(262143)
|
||||
|
||||
val uttClientCacheConfigsV2: UttClientCacheConfigsV2 = UttClientCacheConfigsV2(
|
||||
getTaxonomyConfig = defaultCacheConfigV2,
|
||||
getUttTaxonomyConfig = defaultCacheConfigV2,
|
||||
getLeafIds = defaultCacheConfigV2,
|
||||
getLeafUttEntities = defaultCacheConfigV2
|
||||
)
|
||||
|
||||
// CachedUttClient to use StratoClient
|
||||
new CachedUttClientV2(
|
||||
stratoClient = stratoClient,
|
||||
env = Environment.Prod,
|
||||
cacheConfigs = uttClientCacheConfigsV2,
|
||||
statsReceiver = statsReceiver.scope("cached_utt_client")
|
||||
)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
scala_library(
|
||||
compiler_option_sets = ["fatal_warnings"],
|
||||
platform = "java8",
|
||||
tags = ["bazel-compatible"],
|
||||
dependencies = [
|
||||
"content-recommender/server/src/main/scala/com/twitter/contentrecommender:representation-manager-deps",
|
||||
"frigate/frigate-common/src/main/scala/com/twitter/frigate/common/util",
|
||||
"hermit/hermit-core/src/main/scala/com/twitter/hermit/store/common",
|
||||
"representation-manager/server/src/main/scala/com/twitter/representation_manager/common",
|
||||
"src/scala/com/twitter/simclusters_v2/stores",
|
||||
"src/scala/com/twitter/simclusters_v2/summingbird/stores",
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift-scala",
|
||||
"storage/clients/manhattan/client/src/main/scala",
|
||||
"tweetypie/src/scala/com/twitter/tweetypie/util",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
package com.twitter.representation_manager.store
|
||||
|
||||
import com.twitter.servo.decider.DeciderKeyEnum
|
||||
|
||||
object DeciderConstants {
|
||||
// Deciders inherited from CR and RSX and only used in LegacyRMS
|
||||
// Their value are manipulated by CR and RSX's yml file and their decider dashboard
|
||||
// We will remove them after migration completed
|
||||
val enableLogFavBasedApeEntity20M145KUpdatedEmbeddingCachedStore =
|
||||
"enableLogFavBasedApeEntity20M145KUpdatedEmbeddingCachedStore"
|
||||
|
||||
val enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore =
|
||||
"enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore"
|
||||
|
||||
val enablelogFavBased20M145K2020TweetEmbeddingStoreTimeouts =
|
||||
"enable_log_fav_based_tweet_embedding_20m145k2020_timeouts"
|
||||
val logFavBased20M145K2020TweetEmbeddingStoreTimeoutValueMillis =
|
||||
"log_fav_based_tweet_embedding_20m145k2020_timeout_value_millis"
|
||||
|
||||
val enablelogFavBased20M145KUpdatedTweetEmbeddingStoreTimeouts =
|
||||
"enable_log_fav_based_tweet_embedding_20m145kUpdated_timeouts"
|
||||
val logFavBased20M145KUpdatedTweetEmbeddingStoreTimeoutValueMillis =
|
||||
"log_fav_based_tweet_embedding_20m145kUpdated_timeout_value_millis"
|
||||
|
||||
val enableSimClustersEmbeddingStoreTimeouts = "enable_sim_clusters_embedding_store_timeouts"
|
||||
val simClustersEmbeddingStoreTimeoutValueMillis =
|
||||
"sim_clusters_embedding_store_timeout_value_millis"
|
||||
}
|
||||
|
||||
// Necessary for using servo Gates
|
||||
object DeciderKey extends DeciderKeyEnum {
|
||||
val enableLogFavBasedApeEntity20M145KUpdatedEmbeddingCachedStore: Value = Value(
|
||||
DeciderConstants.enableLogFavBasedApeEntity20M145KUpdatedEmbeddingCachedStore
|
||||
)
|
||||
|
||||
val enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore: Value = Value(
|
||||
DeciderConstants.enableLogFavBasedApeEntity20M145K2020EmbeddingCachedStore
|
||||
)
|
||||
}
|
||||
|
|
@ -0,0 +1,198 @@
|
|||
package com.twitter.representation_manager.store
|
||||
|
||||
import com.twitter.contentrecommender.store.ApeEntityEmbeddingStore
|
||||
import com.twitter.contentrecommender.store.InterestsOptOutStore
|
||||
import com.twitter.contentrecommender.store.SemanticCoreTopicSeedStore
|
||||
import com.twitter.conversions.DurationOps._
|
||||
import com.twitter.escherbird.util.uttclient.CachedUttClientV2
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.store.strato.StratoFetchableStore
|
||||
import com.twitter.frigate.common.util.SeqLongInjection
|
||||
import com.twitter.hermit.store.common.ObservedCachedReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedMemcachedReadableStore
|
||||
import com.twitter.hermit.store.common.ObservedReadableStore
|
||||
import com.twitter.interests.thriftscala.InterestsThriftService
|
||||
import com.twitter.representation_manager.common.MemCacheConfig
|
||||
import com.twitter.representation_manager.common.RepresentationManagerDecider
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.stores.SimClustersEmbeddingStore
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType._
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion._
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.TopicId
|
||||
import com.twitter.simclusters_v2.thriftscala.LocaleEntityId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.tweetypie.util.UserId
|
||||
import javax.inject.Inject
|
||||
|
||||
class TopicSimClustersEmbeddingStore @Inject() (
|
||||
stratoClient: StratoClient,
|
||||
cacheClient: Client,
|
||||
globalStats: StatsReceiver,
|
||||
mhMtlsParams: ManhattanKVClientMtlsParams,
|
||||
rmsDecider: RepresentationManagerDecider,
|
||||
interestService: InterestsThriftService.MethodPerEndpoint,
|
||||
uttClient: CachedUttClientV2) {
|
||||
|
||||
private val stats = globalStats.scope(this.getClass.getSimpleName)
|
||||
private val interestsOptOutStore = InterestsOptOutStore(interestService)
|
||||
|
||||
/**
|
||||
* Note this is NOT an embedding store. It is a list of author account ids we use to represent
|
||||
* topics
|
||||
*/
|
||||
private val semanticCoreTopicSeedStore: ReadableStore[
|
||||
SemanticCoreTopicSeedStore.Key,
|
||||
Seq[UserId]
|
||||
] = {
|
||||
/*
|
||||
Up to 1000 Long seeds per topic/language = 62.5kb per topic/language (worst case)
|
||||
Assume ~10k active topic/languages ~= 650MB (worst case)
|
||||
*/
|
||||
val underlying = new SemanticCoreTopicSeedStore(uttClient, interestsOptOutStore)(
|
||||
stats.scope("semantic_core_topic_seed_store"))
|
||||
|
||||
val memcacheStore = ObservedMemcachedReadableStore.fromCacheClient(
|
||||
backingStore = underlying,
|
||||
cacheClient = cacheClient,
|
||||
ttl = 12.hours)(
|
||||
valueInjection = SeqLongInjection,
|
||||
statsReceiver = stats.scope("topic_producer_seed_store_mem_cache"),
|
||||
keyToString = { k => s"tpss:${k.entityId}_${k.languageCode}" }
|
||||
)
|
||||
|
||||
ObservedCachedReadableStore.from[SemanticCoreTopicSeedStore.Key, Seq[UserId]](
|
||||
store = memcacheStore,
|
||||
ttl = 6.hours,
|
||||
maxKeys = 20e3.toInt,
|
||||
cacheName = "topic_producer_seed_store_cache",
|
||||
windowSize = 5000
|
||||
)(stats.scope("topic_producer_seed_store_cache"))
|
||||
}
|
||||
|
||||
private val favBasedTfgTopicEmbedding20m145k2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/favBasedTFGTopic20M145K2020").mapValues(
|
||||
embedding => SimClustersEmbedding(embedding, truncate = 50).toThrift)
|
||||
.composeKeyMapping[LocaleEntityId] { localeEntityId =>
|
||||
SimClustersEmbeddingId(
|
||||
FavTfgTopic,
|
||||
Model20m145k2020,
|
||||
InternalId.LocaleEntityId(localeEntityId))
|
||||
}
|
||||
|
||||
buildLocaleEntityIdMemCacheStore(rawStore, FavTfgTopic, Model20m145k2020)
|
||||
}
|
||||
|
||||
private val logFavBasedApeEntity20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val apeStore = StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/logFavBasedAPE20M145K2020")
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding, truncate = 50))
|
||||
.composeKeyMapping[UserId]({ id =>
|
||||
SimClustersEmbeddingId(
|
||||
AggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
InternalId.UserId(id))
|
||||
})
|
||||
val rawStore = new ApeEntityEmbeddingStore(
|
||||
semanticCoreSeedStore = semanticCoreTopicSeedStore,
|
||||
aggregatableProducerEmbeddingStore = apeStore,
|
||||
statsReceiver = stats.scope("log_fav_based_ape_entity_2020_embedding_store"))
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding.toThrift, truncate = 50).toThrift)
|
||||
.composeKeyMapping[TopicId] { topicId =>
|
||||
SimClustersEmbeddingId(
|
||||
LogFavBasedKgoApeTopic,
|
||||
Model20m145k2020,
|
||||
InternalId.TopicId(topicId))
|
||||
}
|
||||
|
||||
buildTopicIdMemCacheStore(rawStore, LogFavBasedKgoApeTopic, Model20m145k2020)
|
||||
}
|
||||
|
||||
private def buildTopicIdMemCacheStore(
|
||||
rawStore: ReadableStore[TopicId, ThriftSimClustersEmbedding],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val observedStore: ObservedReadableStore[TopicId, ThriftSimClustersEmbedding] =
|
||||
ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
val storeWithKeyMapping = observedStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.TopicId(topicId)) =>
|
||||
topicId
|
||||
}
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
storeWithKeyMapping,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private def buildLocaleEntityIdMemCacheStore(
|
||||
rawStore: ReadableStore[LocaleEntityId, ThriftSimClustersEmbedding],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val observedStore: ObservedReadableStore[LocaleEntityId, ThriftSimClustersEmbedding] =
|
||||
ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
val storeWithKeyMapping = observedStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.LocaleEntityId(localeEntityId)) =>
|
||||
localeEntityId
|
||||
}
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
storeWithKeyMapping,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private val underlyingStores: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding]
|
||||
] = Map(
|
||||
// Topic Embeddings
|
||||
(FavTfgTopic, Model20m145k2020) -> favBasedTfgTopicEmbedding20m145k2020Store,
|
||||
(LogFavBasedKgoApeTopic, Model20m145k2020) -> logFavBasedApeEntity20M145K2020EmbeddingStore,
|
||||
)
|
||||
|
||||
val topicSimClustersEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
SimClustersEmbeddingStore.buildWithDecider(
|
||||
underlyingStores = underlyingStores,
|
||||
decider = rmsDecider.decider,
|
||||
statsReceiver = stats
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
package com.twitter.representation_manager.store
|
||||
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.hermit.store.common.ObservedReadableStore
|
||||
import com.twitter.representation_manager.common.MemCacheConfig
|
||||
import com.twitter.representation_manager.common.RepresentationManagerDecider
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.common.TweetId
|
||||
import com.twitter.simclusters_v2.stores.SimClustersEmbeddingStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.PersistentTweetEmbeddingStore
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType._
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion._
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import javax.inject.Inject
|
||||
|
||||
class TweetSimClustersEmbeddingStore @Inject() (
|
||||
cacheClient: Client,
|
||||
globalStats: StatsReceiver,
|
||||
mhMtlsParams: ManhattanKVClientMtlsParams,
|
||||
rmsDecider: RepresentationManagerDecider) {
|
||||
|
||||
private val stats = globalStats.scope(this.getClass.getSimpleName)
|
||||
|
||||
val logFavBasedLongestL2Tweet20M145KUpdatedEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.longestL2NormTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145kUpdatedDataset,
|
||||
stats
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
buildMemCacheStore(rawStore, LogFavLongestL2EmbeddingTweet, Model20m145kUpdated)
|
||||
}
|
||||
|
||||
val logFavBasedLongestL2Tweet20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.longestL2NormTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset,
|
||||
stats
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
buildMemCacheStore(rawStore, LogFavLongestL2EmbeddingTweet, Model20m145k2020)
|
||||
}
|
||||
|
||||
val logFavBased20M145KUpdatedTweetEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.mostRecentTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145kUpdatedDataset,
|
||||
stats
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
buildMemCacheStore(rawStore, LogFavBasedTweet, Model20m145kUpdated)
|
||||
}
|
||||
|
||||
val logFavBased20M145K2020TweetEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
PersistentTweetEmbeddingStore
|
||||
.mostRecentTweetEmbeddingStoreManhattan(
|
||||
mhMtlsParams,
|
||||
PersistentTweetEmbeddingStore.LogFavBased20m145k2020Dataset,
|
||||
stats
|
||||
).mapValues(_.toThrift)
|
||||
|
||||
buildMemCacheStore(rawStore, LogFavBasedTweet, Model20m145k2020)
|
||||
}
|
||||
|
||||
private def buildMemCacheStore(
|
||||
rawStore: ReadableStore[TweetId, ThriftSimClustersEmbedding],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val observedStore: ObservedReadableStore[TweetId, ThriftSimClustersEmbedding] =
|
||||
ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
val storeWithKeyMapping = observedStore.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.TweetId(tweetId)) =>
|
||||
tweetId
|
||||
}
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
storeWithKeyMapping,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private val underlyingStores: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding]
|
||||
] = Map(
|
||||
// Tweet Embeddings
|
||||
(LogFavBasedTweet, Model20m145kUpdated) -> logFavBased20M145KUpdatedTweetEmbeddingStore,
|
||||
(LogFavBasedTweet, Model20m145k2020) -> logFavBased20M145K2020TweetEmbeddingStore,
|
||||
(
|
||||
LogFavLongestL2EmbeddingTweet,
|
||||
Model20m145kUpdated) -> logFavBasedLongestL2Tweet20M145KUpdatedEmbeddingStore,
|
||||
(
|
||||
LogFavLongestL2EmbeddingTweet,
|
||||
Model20m145k2020) -> logFavBasedLongestL2Tweet20M145K2020EmbeddingStore,
|
||||
)
|
||||
|
||||
val tweetSimClustersEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
SimClustersEmbeddingStore.buildWithDecider(
|
||||
underlyingStores = underlyingStores,
|
||||
decider = rmsDecider.decider,
|
||||
statsReceiver = stats
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,602 @@
|
|||
package com.twitter.representation_manager.store
|
||||
|
||||
import com.twitter.contentrecommender.twistly
|
||||
import com.twitter.finagle.memcached.Client
|
||||
import com.twitter.finagle.stats.StatsReceiver
|
||||
import com.twitter.frigate.common.store.strato.StratoFetchableStore
|
||||
import com.twitter.hermit.store.common.ObservedReadableStore
|
||||
import com.twitter.representation_manager.common.MemCacheConfig
|
||||
import com.twitter.representation_manager.common.RepresentationManagerDecider
|
||||
import com.twitter.simclusters_v2.common.ModelVersions
|
||||
import com.twitter.simclusters_v2.common.SimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.stores.SimClustersEmbeddingStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.ProducerClusterEmbeddingReadableStores
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore.getStore
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore.modelVersionToDatasetMap
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore.knownModelVersions
|
||||
import com.twitter.simclusters_v2.summingbird.stores.UserInterestedInReadableStore.toSimClustersEmbedding
|
||||
import com.twitter.simclusters_v2.thriftscala.ClustersUserIsInterestedIn
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType
|
||||
import com.twitter.simclusters_v2.thriftscala.EmbeddingType._
|
||||
import com.twitter.simclusters_v2.thriftscala.InternalId
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion
|
||||
import com.twitter.simclusters_v2.thriftscala.ModelVersion._
|
||||
import com.twitter.simclusters_v2.thriftscala.SimClustersEmbeddingId
|
||||
import com.twitter.simclusters_v2.thriftscala.{SimClustersEmbedding => ThriftSimClustersEmbedding}
|
||||
import com.twitter.storage.client.manhattan.kv.ManhattanKVClientMtlsParams
|
||||
import com.twitter.storehaus.ReadableStore
|
||||
import com.twitter.storehaus_internal.manhattan.Apollo
|
||||
import com.twitter.storehaus_internal.manhattan.ManhattanCluster
|
||||
import com.twitter.strato.client.{Client => StratoClient}
|
||||
import com.twitter.strato.thrift.ScroogeConvImplicits._
|
||||
import com.twitter.tweetypie.util.UserId
|
||||
import com.twitter.util.Future
|
||||
import javax.inject.Inject
|
||||
|
||||
class UserSimClustersEmbeddingStore @Inject() (
|
||||
stratoClient: StratoClient,
|
||||
cacheClient: Client,
|
||||
globalStats: StatsReceiver,
|
||||
mhMtlsParams: ManhattanKVClientMtlsParams,
|
||||
rmsDecider: RepresentationManagerDecider) {
|
||||
|
||||
private val stats = globalStats.scope(this.getClass.getSimpleName)
|
||||
|
||||
private val favBasedProducer20M145KUpdatedEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = ProducerClusterEmbeddingReadableStores
|
||||
.getProducerTopKSimClustersEmbeddingsStore(
|
||||
mhMtlsParams
|
||||
).mapValues { topSimClustersWithScore =>
|
||||
ThriftSimClustersEmbedding(topSimClustersWithScore.topClusters)
|
||||
}.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.UserId(userId)) =>
|
||||
userId
|
||||
}
|
||||
|
||||
buildMemCacheStore(rawStore, FavBasedProducer, Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val favBasedProducer20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = ProducerClusterEmbeddingReadableStores
|
||||
.getProducerTopKSimClusters2020EmbeddingsStore(
|
||||
mhMtlsParams
|
||||
).mapValues { topSimClustersWithScore =>
|
||||
ThriftSimClustersEmbedding(topSimClustersWithScore.topClusters)
|
||||
}.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.UserId(userId)) =>
|
||||
userId
|
||||
}
|
||||
|
||||
buildMemCacheStore(rawStore, FavBasedProducer, Model20m145k2020)
|
||||
}
|
||||
|
||||
private val followBasedProducer20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = ProducerClusterEmbeddingReadableStores
|
||||
.getProducerTopKSimClustersEmbeddingsByFollowStore(
|
||||
mhMtlsParams
|
||||
).mapValues { topSimClustersWithScore =>
|
||||
ThriftSimClustersEmbedding(topSimClustersWithScore.topClusters)
|
||||
}.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(_, _, InternalId.UserId(userId)) =>
|
||||
userId
|
||||
}
|
||||
|
||||
buildMemCacheStore(rawStore, FollowBasedProducer, Model20m145k2020)
|
||||
}
|
||||
|
||||
private val logFavBasedApe20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/logFavBasedAPE20M145K2020")
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding, truncate = 50).toThrift)
|
||||
|
||||
buildMemCacheStore(rawStore, AggregatableLogFavBasedProducer, Model20m145k2020)
|
||||
}
|
||||
|
||||
private val rawRelaxedLogFavBasedApe20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
ThriftSimClustersEmbedding
|
||||
] = {
|
||||
StratoFetchableStore
|
||||
.withUnitView[SimClustersEmbeddingId, ThriftSimClustersEmbedding](
|
||||
stratoClient,
|
||||
"recommendations/simclusters_v2/embeddings/logFavBasedAPERelaxedFavEngagementThreshold20M145K2020")
|
||||
.mapValues(embedding => SimClustersEmbedding(embedding, truncate = 50).toThrift)
|
||||
}
|
||||
|
||||
private val relaxedLogFavBasedApe20M145K2020EmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(
|
||||
rawRelaxedLogFavBasedApe20M145K2020EmbeddingStore,
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val relaxedLogFavBasedApe20m145kUpdatedEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = rawRelaxedLogFavBasedApe20M145K2020EmbeddingStore
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145kUpdated,
|
||||
internalId) =>
|
||||
SimClustersEmbeddingId(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020,
|
||||
internalId)
|
||||
}
|
||||
|
||||
buildMemCacheStore(rawStore, RelaxedAggregatableLogFavBasedProducer, Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val logFavBasedInterestedInFromAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultIIAPESimClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val followBasedInterestedInFromAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultIIAPESimClustersEmbeddingStoreWithMtls,
|
||||
FollowBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val favBasedUserInterestedIn20M145KUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultSimClustersEmbeddingStoreWithMtls,
|
||||
FavBasedUserInterestedIn,
|
||||
Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val favBasedUserInterestedIn20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultSimClustersEmbeddingStoreWithMtls,
|
||||
FavBasedUserInterestedIn,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val followBasedUserInterestedIn20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultSimClustersEmbeddingStoreWithMtls,
|
||||
FollowBasedUserInterestedIn,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val logFavBasedUserInterestedIn20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultSimClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedIn,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val favBasedUserInterestedInFromPE20M145KUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultIIPESimClustersEmbeddingStoreWithMtls,
|
||||
FavBasedUserInterestedInFromPE,
|
||||
Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val twistlyUserInterestedInStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
ThriftSimClustersEmbedding
|
||||
] = {
|
||||
val interestedIn20M145KUpdatedStore = {
|
||||
UserInterestedInReadableStore.defaultStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145KUpdated
|
||||
)
|
||||
}
|
||||
val interestedIn20M145K2020Store = {
|
||||
UserInterestedInReadableStore.defaultStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145K2020
|
||||
)
|
||||
}
|
||||
val interestedInFromPE20M145KUpdatedStore = {
|
||||
UserInterestedInReadableStore.defaultIIPEStoreWithMtls(
|
||||
mhMtlsParams,
|
||||
modelVersion = ModelVersions.Model20M145KUpdated)
|
||||
}
|
||||
val simClustersInterestedInStore: ReadableStore[
|
||||
(UserId, ModelVersion),
|
||||
ClustersUserIsInterestedIn
|
||||
] = {
|
||||
new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] {
|
||||
override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = {
|
||||
k match {
|
||||
case (userId, Model20m145kUpdated) =>
|
||||
interestedIn20M145KUpdatedStore.get(userId)
|
||||
case (userId, Model20m145k2020) =>
|
||||
interestedIn20M145K2020Store.get(userId)
|
||||
case _ =>
|
||||
Future.None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
val simClustersInterestedInFromProducerEmbeddingsStore: ReadableStore[
|
||||
(UserId, ModelVersion),
|
||||
ClustersUserIsInterestedIn
|
||||
] = {
|
||||
new ReadableStore[(UserId, ModelVersion), ClustersUserIsInterestedIn] {
|
||||
override def get(k: (UserId, ModelVersion)): Future[Option[ClustersUserIsInterestedIn]] = {
|
||||
k match {
|
||||
case (userId, ModelVersion.Model20m145kUpdated) =>
|
||||
interestedInFromPE20M145KUpdatedStore.get(userId)
|
||||
case _ =>
|
||||
Future.None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
new twistly.interestedin.EmbeddingStore(
|
||||
interestedInStore = simClustersInterestedInStore,
|
||||
interestedInFromProducerEmbeddingStore = simClustersInterestedInFromProducerEmbeddingsStore,
|
||||
statsReceiver = stats
|
||||
).mapValues(_.toThrift)
|
||||
}
|
||||
|
||||
private val userNextInterestedIn20m145k2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildUserInterestedInStore(
|
||||
UserInterestedInReadableStore.defaultNextInterestedInStoreWithMtls,
|
||||
UserNextInterestedIn,
|
||||
Model20m145k2020)
|
||||
}
|
||||
|
||||
private val filteredUserInterestedIn20m145kUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(twistlyUserInterestedInStore, FilteredUserInterestedIn, Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val filteredUserInterestedIn20m145k2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(twistlyUserInterestedInStore, FilteredUserInterestedIn, Model20m145k2020)
|
||||
}
|
||||
|
||||
private val filteredUserInterestedInFromPE20m145kUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(
|
||||
twistlyUserInterestedInStore,
|
||||
FilteredUserInterestedInFromPE,
|
||||
Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val unfilteredUserInterestedIn20m145kUpdatedStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(
|
||||
twistlyUserInterestedInStore,
|
||||
UnfilteredUserInterestedIn,
|
||||
Model20m145kUpdated)
|
||||
}
|
||||
|
||||
private val unfilteredUserInterestedIn20m145k2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
buildMemCacheStore(twistlyUserInterestedInStore, UnfilteredUserInterestedIn, Model20m145k2020)
|
||||
}
|
||||
|
||||
// [Experimental] User InterestedIn, generated by aggregating IIAPE embedding from AddressBook
|
||||
|
||||
private val logFavBasedInterestedMaxpoolingAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_maxpooling"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
private val logFavBasedInterestedAverageAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_average"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedAverageAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
private val logFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_booktype_maxpooling"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
private val logFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_largestdim_maxpooling"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
private val logFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_louvain_maxpooling"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
private val logFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE20M145K2020Store: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val datasetName = "addressbook_sims_embedding_iiape_connected_maxpooling"
|
||||
val appId = "wtf_embedding_apollo"
|
||||
buildUserInterestedInStoreGeneric(
|
||||
simClustersEmbeddingStoreWithMtls,
|
||||
LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020,
|
||||
datasetName = datasetName,
|
||||
appId = appId,
|
||||
manhattanCluster = Apollo
|
||||
)
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper func to build a readable store for some UserInterestedIn embeddings with
|
||||
* 1. A storeFunc from UserInterestedInReadableStore
|
||||
* 2. EmbeddingType
|
||||
* 3. ModelVersion
|
||||
* 4. MemCacheConfig
|
||||
* */
|
||||
private def buildUserInterestedInStore(
|
||||
storeFunc: (ManhattanKVClientMtlsParams, EmbeddingType, ModelVersion) => ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore = storeFunc(mhMtlsParams, embeddingType, modelVersion)
|
||||
.mapValues(_.toThrift)
|
||||
val observedStore = ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
observedStore,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private def buildUserInterestedInStoreGeneric(
|
||||
storeFunc: (ManhattanKVClientMtlsParams, EmbeddingType, ModelVersion, String, String,
|
||||
ManhattanCluster) => ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion,
|
||||
datasetName: String,
|
||||
appId: String,
|
||||
manhattanCluster: ManhattanCluster
|
||||
): ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
val rawStore =
|
||||
storeFunc(mhMtlsParams, embeddingType, modelVersion, datasetName, appId, manhattanCluster)
|
||||
.mapValues(_.toThrift)
|
||||
val observedStore = ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
observedStore,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private def simClustersEmbeddingStoreWithMtls(
|
||||
mhMtlsParams: ManhattanKVClientMtlsParams,
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion,
|
||||
datasetName: String,
|
||||
appId: String,
|
||||
manhattanCluster: ManhattanCluster
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
|
||||
if (!modelVersionToDatasetMap.contains(ModelVersions.toKnownForModelVersion(modelVersion))) {
|
||||
throw new IllegalArgumentException(
|
||||
"Unknown model version: " + modelVersion + ". Known model versions: " + knownModelVersions)
|
||||
}
|
||||
getStore(appId, mhMtlsParams, datasetName, manhattanCluster)
|
||||
.composeKeyMapping[SimClustersEmbeddingId] {
|
||||
case SimClustersEmbeddingId(theEmbeddingType, theModelVersion, InternalId.UserId(userId))
|
||||
if theEmbeddingType == embeddingType && theModelVersion == modelVersion =>
|
||||
userId
|
||||
}.mapValues(toSimClustersEmbedding(_, embeddingType))
|
||||
}
|
||||
|
||||
private def buildMemCacheStore(
|
||||
rawStore: ReadableStore[SimClustersEmbeddingId, ThriftSimClustersEmbedding],
|
||||
embeddingType: EmbeddingType,
|
||||
modelVersion: ModelVersion
|
||||
): ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding] = {
|
||||
val observedStore = ObservedReadableStore(
|
||||
store = rawStore
|
||||
)(stats.scope(embeddingType.name).scope(modelVersion.name))
|
||||
|
||||
MemCacheConfig.buildMemCacheStoreForSimClustersEmbedding(
|
||||
observedStore,
|
||||
cacheClient,
|
||||
embeddingType,
|
||||
modelVersion,
|
||||
stats
|
||||
)
|
||||
}
|
||||
|
||||
private val underlyingStores: Map[
|
||||
(EmbeddingType, ModelVersion),
|
||||
ReadableStore[SimClustersEmbeddingId, SimClustersEmbedding]
|
||||
] = Map(
|
||||
// KnownFor Embeddings
|
||||
(FavBasedProducer, Model20m145kUpdated) -> favBasedProducer20M145KUpdatedEmbeddingStore,
|
||||
(FavBasedProducer, Model20m145k2020) -> favBasedProducer20M145K2020EmbeddingStore,
|
||||
(FollowBasedProducer, Model20m145k2020) -> followBasedProducer20M145K2020EmbeddingStore,
|
||||
(AggregatableLogFavBasedProducer, Model20m145k2020) -> logFavBasedApe20M145K2020EmbeddingStore,
|
||||
(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145kUpdated) -> relaxedLogFavBasedApe20m145kUpdatedEmbeddingStore,
|
||||
(
|
||||
RelaxedAggregatableLogFavBasedProducer,
|
||||
Model20m145k2020) -> relaxedLogFavBasedApe20M145K2020EmbeddingStore,
|
||||
// InterestedIn Embeddings
|
||||
(
|
||||
LogFavBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020) -> logFavBasedInterestedInFromAPE20M145K2020Store,
|
||||
(
|
||||
FollowBasedUserInterestedInFromAPE,
|
||||
Model20m145k2020) -> followBasedInterestedInFromAPE20M145K2020Store,
|
||||
(FavBasedUserInterestedIn, Model20m145kUpdated) -> favBasedUserInterestedIn20M145KUpdatedStore,
|
||||
(FavBasedUserInterestedIn, Model20m145k2020) -> favBasedUserInterestedIn20M145K2020Store,
|
||||
(FollowBasedUserInterestedIn, Model20m145k2020) -> followBasedUserInterestedIn20M145K2020Store,
|
||||
(LogFavBasedUserInterestedIn, Model20m145k2020) -> logFavBasedUserInterestedIn20M145K2020Store,
|
||||
(
|
||||
FavBasedUserInterestedInFromPE,
|
||||
Model20m145kUpdated) -> favBasedUserInterestedInFromPE20M145KUpdatedStore,
|
||||
(FilteredUserInterestedIn, Model20m145kUpdated) -> filteredUserInterestedIn20m145kUpdatedStore,
|
||||
(FilteredUserInterestedIn, Model20m145k2020) -> filteredUserInterestedIn20m145k2020Store,
|
||||
(
|
||||
FilteredUserInterestedInFromPE,
|
||||
Model20m145kUpdated) -> filteredUserInterestedInFromPE20m145kUpdatedStore,
|
||||
(
|
||||
UnfilteredUserInterestedIn,
|
||||
Model20m145kUpdated) -> unfilteredUserInterestedIn20m145kUpdatedStore,
|
||||
(UnfilteredUserInterestedIn, Model20m145k2020) -> unfilteredUserInterestedIn20m145k2020Store,
|
||||
(UserNextInterestedIn, Model20m145k2020) -> userNextInterestedIn20m145k2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedInterestedMaxpoolingAddressBookFromIIAPE20M145K2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedAverageAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedInterestedAverageAddressBookFromIIAPE20M145K2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedUserInterestedBooktypeMaxpoolingAddressBookFromIIAPE20M145K2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedUserInterestedLargestDimMaxpoolingAddressBookFromIIAPE20M145K2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedUserInterestedLouvainMaxpoolingAddressBookFromIIAPE20M145K2020Store,
|
||||
(
|
||||
LogFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE,
|
||||
Model20m145k2020) -> logFavBasedUserInterestedConnectedMaxpoolingAddressBookFromIIAPE20M145K2020Store,
|
||||
)
|
||||
|
||||
val userSimClustersEmbeddingStore: ReadableStore[
|
||||
SimClustersEmbeddingId,
|
||||
SimClustersEmbedding
|
||||
] = {
|
||||
SimClustersEmbeddingStore.buildWithDecider(
|
||||
underlyingStores = underlyingStores,
|
||||
decider = rmsDecider.decider,
|
||||
statsReceiver = stats
|
||||
)
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
create_thrift_libraries(
|
||||
base_name = "thrift",
|
||||
sources = [
|
||||
"com/twitter/representation_manager/service.thrift",
|
||||
],
|
||||
platform = "java8",
|
||||
tags = [
|
||||
"bazel-compatible",
|
||||
],
|
||||
dependency_roots = [
|
||||
"src/thrift/com/twitter/simclusters_v2:simclusters_v2-thrift",
|
||||
],
|
||||
generate_languages = [
|
||||
"java",
|
||||
"scala",
|
||||
"strato",
|
||||
],
|
||||
)
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
namespace java com.twitter.representation_manager.thriftjava
|
||||
#@namespace scala com.twitter.representation_manager.thriftscala
|
||||
#@namespace strato com.twitter.representation_manager
|
||||
|
||||
include "com/twitter/simclusters_v2/online_store.thrift"
|
||||
include "com/twitter/simclusters_v2/identifier.thrift"
|
||||
|
||||
/**
|
||||
* A uniform column view for all kinds of SimClusters based embeddings.
|
||||
**/
|
||||
struct SimClustersEmbeddingView {
|
||||
1: required identifier.EmbeddingType embeddingType
|
||||
2: required online_store.ModelVersion modelVersion
|
||||
}(persisted = 'false', hasPersonalData = 'false')
|
||||
Loading…
Reference in New Issue