badjware
/
the-algorithm
mirror of https://github.com/twitter/the-algorithm.git
1
0
Fork 0
Code Releases Activity

Change if refactored to match

This commit is contained in:
denon1 2023-04-01 02:04:07 +02:00
parent 573fab9526
commit 376f35f08f
1 changed files with 23 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
src/scala/com/twitter/simclusters_v2/scalding/KnownForSources.scala
View File

@ -102,28 +102,30 @@ object KnownForSources {
TypedPipe TypedPipe
.from(TextLine(textFile)) .from(TextLine(textFile))
.flatMap { str => .flatMap { str =>
if (str.startsWith("#")) {none} str match {
try { case s"#$_" => none
val tokens = str.trim.split("\\s+") case _ => try {
val userId = tokens(0).toLong val tokens = str.trim.split("\\s+")
(1 until tokens.length).foldRight(Array.newBuilder[(Int, Float)])((i, r) => { val userId = tokens(0).toLong
val Array(cIdStr, scoreStr) = tokens(i).split(":") (1 until tokens.length).foldRight(Array.newBuilder[(Int, Float)])((i, r) => {
val clusterId = cIdStr.toInt val Array(cIdStr, scoreStr) = tokens(i).split(":")
val score = scoreStr.toFloat val clusterId = cIdStr.toInt
val newEntry = (clusterId, score) val score = scoreStr.toFloat
r += newEntry val newEntry = (clusterId, score)
}).result() match { r += newEntry
case (res) if res.nonEmpty => Some((userId, res.result())) }).result() match {
_ => none case (res) if res.nonEmpty => Some((userId, res.result()))
_ => none
}
}
catch {
case ex: Throwable =>
log.warning(
s"Error while loading knownFor from $textFile for line <$str>: " +
ex.getMessage
)
None
} }
}
catch {
case ex: Throwable =>
log.warning(
s"Error while loading knownFor from $textFile for line <$str>: " +
ex.getMessage
)
None
} }
} }
} }