diff options
Diffstat (limited to 'Scala/objsets')
26 files changed, 2903 insertions, 0 deletions
diff --git a/Scala/objsets/.classpath b/Scala/objsets/.classpath new file mode 100644 index 0000000..572103c --- /dev/null +++ b/Scala/objsets/.classpath @@ -0,0 +1,12 @@ +<classpath> + <classpathentry output="target/scala-2.10/classes" path="src/main/scala" kind="src"></classpathentry> + <classpathentry output="target/scala-2.10/test-classes" path="src/test/scala" kind="src"></classpathentry> + <classpathentry kind="con" path="org.scala-ide.sdt.launching.SCALA_CONTAINER"></classpathentry> + <classpathentry path="./lib_managed/jars/org.scalatest/scalatest_2.10/scalatest_2.10-1.9.1.jar" kind="lib"></classpathentry> + <classpathentry path="./lib_managed/jars/org.scala-lang/scala-actors/scala-actors-2.10.0.jar" kind="lib"></classpathentry> + <classpathentry path="./lib_managed/jars/org.scala-lang/scala-reflect/scala-reflect-2.10.0.jar" kind="lib"></classpathentry> + <classpathentry path="./lib_managed/jars/junit/junit/junit-4.10.jar" kind="lib"></classpathentry> + <classpathentry path="./lib_managed/jars/org.hamcrest/hamcrest-core/hamcrest-core-1.1.jar" kind="lib"></classpathentry> + <classpathentry path="org.eclipse.jdt.launching.JRE_CONTAINER" kind="con"></classpathentry> + <classpathentry path="bin" kind="output"></classpathentry> +</classpath>
\ No newline at end of file diff --git a/Scala/objsets/.project b/Scala/objsets/.project new file mode 100644 index 0000000..2c1b75d --- /dev/null +++ b/Scala/objsets/.project @@ -0,0 +1,12 @@ +<projectDescription> + <name>progfun-objsets</name> + <buildSpec> + <buildCommand> + <name>org.scala-ide.sdt.core.scalabuilder</name> + </buildCommand> + </buildSpec> + <natures> + <nature>org.scala-ide.sdt.core.scalanature</nature> + <nature>org.eclipse.jdt.core.javanature</nature> + </natures> +</projectDescription>
\ No newline at end of file diff --git a/Scala/objsets/.settings/org.scala-ide.sdt.core.prefs b/Scala/objsets/.settings/org.scala-ide.sdt.core.prefs new file mode 100644 index 0000000..3293297 --- /dev/null +++ b/Scala/objsets/.settings/org.scala-ide.sdt.core.prefs @@ -0,0 +1,5 @@ +#Generated by sbteclipse +#Fri Apr 12 08:09:40 CEST 2013 +deprecation=true +feature=true +scala.compiler.useProjectSettings=true diff --git a/Scala/objsets/build.sbt b/Scala/objsets/build.sbt new file mode 100644 index 0000000..470f461 --- /dev/null +++ b/Scala/objsets/build.sbt @@ -0,0 +1,110 @@ +name <<= submitProjectName(pname => "progfun-"+ pname) + +version := "1.0.0" + +scalaVersion := "2.10.1" + +scalacOptions ++= Seq("-deprecation", "-feature") + +libraryDependencies += "org.scalatest" %% "scalatest" % "1.9.1" % "test" + +libraryDependencies += "junit" % "junit" % "4.10" % "test" + +// This setting defines the project to which a solution is submitted. When creating a +// handout, the 'createHandout' task will make sure that its value is correct. +submitProjectName := "objsets" + +// See documentation in ProgFunBuild.scala +projectDetailsMap := { +val currentCourseId = "progfun-002" +Map( + "example" -> ProjectDetails( + packageName = "example", + assignmentPartId = "fTzFogNl", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "recfun" -> ProjectDetails( + packageName = "recfun", + assignmentPartId = "3Rarn9Ki", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "funsets" -> ProjectDetails( + packageName = "funsets", + assignmentPartId = "fBXOL6Qd", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "objsets" -> ProjectDetails( + packageName = "objsets", + assignmentPartId = "95dMMEz7", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "patmat" -> ProjectDetails( + packageName = "patmat", + assignmentPartId = "3gPmpcif", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "forcomp" -> ProjectDetails( + packageName = "forcomp", + assignmentPartId = "fG1oZGIO", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId), + "streams" -> ProjectDetails( + packageName = "streams", + assignmentPartId = "CWKgCFCi", + maxScore = 10d, + styleScoreRatio = 0.2, + courseId=currentCourseId)//, + // "simulations" -> ProjectDetails( + // packageName = "simulations", + // assignmentPartId = "iYs4GARk", + // maxScore = 10d, + // styleScoreRatio = 0.2, + // courseId="progfun2-001"), + // "interpreter" -> ProjectDetails( + // packageName = "interpreter", + // assignmentPartId = "1SZhe1Ut", + // maxScore = 10d, + // styleScoreRatio = 0.2, + // courseId="progfun2-001") +) +} + +// Files that we hand out to the students +handoutFiles <<= (baseDirectory, projectDetailsMap, commonSourcePackages) map { (basedir, detailsMap, commonSrcs) => + (projectName: String) => { + val details = detailsMap.getOrElse(projectName, sys.error("Unknown project name: "+ projectName)) + val commonFiles = (PathFinder.empty /: commonSrcs)((files, pkg) => + files +++ (basedir / "src" / "main" / "scala" / pkg ** "*.scala") + ) + (basedir / "src" / "main" / "scala" / details.packageName ** "*.scala") +++ + commonFiles +++ + (basedir / "src" / "main" / "resources" / details.packageName ** "*") +++ + (basedir / "src" / "test" / "scala" / details.packageName ** "*.scala") +++ + (basedir / "build.sbt") +++ + (basedir / "project" / "build.properties") +++ + (basedir / "project" ** ("*.scala" || "*.sbt")) +++ + (basedir / "project" / "scalastyle_config.xml") +++ + (basedir / "lib_managed" ** "*.jar") +++ + (basedir * (".classpath" || ".project")) +++ + (basedir / ".settings" / "org.scala-ide.sdt.core.prefs") + } +} + +// This setting allows to restrict the source files that are compiled and tested +// to one specific project. It should be either the empty string, in which case all +// projects are included, or one of the project names from the projectDetailsMap. +currentProject := "" + +// Packages in src/main/scala that are used in every project. Included in every +// handout, submission. +commonSourcePackages += "common" + +// Packages in src/test/scala that are used for grading projects. Always included +// compiling tests, grading a project. +gradingTestPackages += "grading" diff --git a/Scala/objsets/lib_managed/jars/junit/junit/junit-4.10.jar b/Scala/objsets/lib_managed/jars/junit/junit/junit-4.10.jar Binary files differnew file mode 100644 index 0000000..954851e --- /dev/null +++ b/Scala/objsets/lib_managed/jars/junit/junit/junit-4.10.jar diff --git a/Scala/objsets/lib_managed/jars/org.hamcrest/hamcrest-core/hamcrest-core-1.1.jar b/Scala/objsets/lib_managed/jars/org.hamcrest/hamcrest-core/hamcrest-core-1.1.jar Binary files differnew file mode 100644 index 0000000..e5149be --- /dev/null +++ b/Scala/objsets/lib_managed/jars/org.hamcrest/hamcrest-core/hamcrest-core-1.1.jar diff --git a/Scala/objsets/lib_managed/jars/org.scala-lang/scala-actors/scala-actors-2.10.0.jar b/Scala/objsets/lib_managed/jars/org.scala-lang/scala-actors/scala-actors-2.10.0.jar Binary files differnew file mode 100644 index 0000000..bb4600c --- /dev/null +++ b/Scala/objsets/lib_managed/jars/org.scala-lang/scala-actors/scala-actors-2.10.0.jar diff --git a/Scala/objsets/lib_managed/jars/org.scala-lang/scala-reflect/scala-reflect-2.10.0.jar b/Scala/objsets/lib_managed/jars/org.scala-lang/scala-reflect/scala-reflect-2.10.0.jar Binary files differnew file mode 100644 index 0000000..6489599 --- /dev/null +++ b/Scala/objsets/lib_managed/jars/org.scala-lang/scala-reflect/scala-reflect-2.10.0.jar diff --git a/Scala/objsets/lib_managed/jars/org.scalatest/scalatest_2.10/scalatest_2.10-1.9.1.jar b/Scala/objsets/lib_managed/jars/org.scalatest/scalatest_2.10/scalatest_2.10-1.9.1.jar Binary files differnew file mode 100644 index 0000000..6be20e6 --- /dev/null +++ b/Scala/objsets/lib_managed/jars/org.scalatest/scalatest_2.10/scalatest_2.10-1.9.1.jar diff --git a/Scala/objsets/project/CourseraHttp.scala b/Scala/objsets/project/CourseraHttp.scala new file mode 100644 index 0000000..5f55b12 --- /dev/null +++ b/Scala/objsets/project/CourseraHttp.scala @@ -0,0 +1,223 @@ +import dispatch.{Request, Http, NoLogging, StatusCode, url} +import cc.spray.json.{JsNull, JsonParser, DefaultJsonProtocol, JsValue} +import RichJsValue._ +import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.codec.binary.{Hex, Base64} +import java.io.{IOException, File, FileInputStream} +import scalaz.Scalaz.{mkIdentity, ValidationNEL} + +import Settings._ + +case class JsonSubmission(api_state: String, user_info: JsValue, submission_metadata: JsValue, solutions: JsValue, submission_encoding: String, submission: String) +//case class JsonQueueResult(submission: JsonSubmission) +object SubmitJsonProtocol extends DefaultJsonProtocol { + implicit val jsonSubmissionFormat = jsonFormat6(JsonSubmission) +// implicit val jsonQueueResultFormat = jsonFormat1(JsonQueueResult) +} + +object CourseraHttp { + private lazy val http = new Http with NoLogging + + private def executeRequest[T](req: Request)(parse: String => ValidationNEL[String, T]): ValidationNEL[String, T] = { + try { + http(req >- { res => + parse(res) + }) + } catch { + case ex: IOException => + ("Connection failed\n"+ ex.toString()).failNel + case StatusCode(code, message) => + ("HTTP failed with status "+ code +"\n"+ message).failNel + } + } + + + /****************************** + * SUBMITTING + */ + + def getChallenge(email: String, submitProject: ProjectDetails): ValidationNEL[String, Challenge] = { + val baseReq = url(challengeUrl(submitProject.courseId)) + val withArgs = baseReq << Map("email_address" -> email, + "assignment_part_sid" -> submitProject.assignmentPartId, + "response_encoding" -> "delim") + + executeRequest(withArgs) { res => + // example result. there might be an `aux_data` value at the end. + // |email_address|a@b.com|challenge_key|XXYYXXYYXXYY|state|XXYYXXYYXXYY|challenge_aux_data| + val parts = res.split('|').filterNot(_.isEmpty) + if (parts.length < 7) + ("Unexpected challenge format: \n"+ res).failNel + else + Challenge(parts(1), parts(3), parts(5)).successNel + } + } + + def submitSolution(sourcesJar: File, submitProject: ProjectDetails, challenge: Challenge, chResponse: String): ValidationNEL[String, String] = { + val fileLength = sourcesJar.length() + if (!sourcesJar.exists()) { + ("Sources jar archive does not exist\n"+ sourcesJar.getAbsolutePath).failNel + } else if (fileLength == 0l) { + ("Sources jar archive is empty\n"+ sourcesJar.getAbsolutePath).failNel + } else if (fileLength > maxSubmitFileSize) { + ("Sources jar archive is too big. Allowed size: "+ + maxSubmitFileSize +" bytes, found "+ fileLength +" bytes.\n"+ + sourcesJar.getAbsolutePath).failNel + } else { + val bytes = new Array[Byte](fileLength.toInt) + val sizeRead = try { + val is = new FileInputStream(sourcesJar) + val read = is.read(bytes) + is.close() + read + } catch { + case ex: IOException => + ("Failed to read sources jar archive\n"+ ex.toString()).failNel + } + if (sizeRead != bytes.length) { + ("Failed to read the sources jar archive, size read: "+ sizeRead).failNel + } else { + val fileData = encodeBase64(bytes) + val baseReq = url(submitUrl(submitProject.courseId)) + val withArgs = baseReq << Map("assignment_part_sid" -> submitProject.assignmentPartId, + "email_address" -> challenge.email, + "submission" -> fileData, + "submission_aux" -> "", + "challenge_response" -> chResponse, + "state" -> challenge.state) + executeRequest(withArgs) { res => + // the API returns HTTP 200 even if there are failures, how impolite... + if (res.contains("Your submission has been accepted")) + res.successNel + else + res.failNel + } + } + } + } + + def challengeResponse(challenge: Challenge, otPassword: String): String = + shaHexDigest(challenge.challengeKey + otPassword) + + + /******************************** + * DOWNLOADING SUBMISSIONS + */ + + // def downloadFromQueue(queue: String, targetJar: File, apiKey: String): ValidationNEL[String, QueueResult] = { + // val baseReq = url(Settings.submitQueueUrl) + // val withArgsAndHeader = baseReq << Map("queue" -> queue) <:< Map("X-api-key" -> apiKey) + + // executeRequest(withArgsAndHeader) { res => + // extractJson(res, targetJar) + // } + // } + + def readJsonFile(jsonFile: File, targetJar: File): ValidationNEL[String, QueueResult] = { + extractJson(sbt.IO.read(jsonFile), targetJar) + } + + def extractJson(jsonData: String, targetJar: File): ValidationNEL[String, QueueResult] = { + import SubmitJsonProtocol._ + for { + jsonSubmission <- { + try { + val parsed = JsonParser(jsonData) + val submission = parsed \ "submission" + if (submission == JsNull) { + ("Nothing to grade, queue is empty.").failNel + } else { + submission.convertTo[JsonSubmission].successNel + } + } catch { + case e: Exception => + ("Could not parse submission\n"+ jsonData +"\n"+ fullExceptionString(e)).failNel + } + } + queueResult <- { + val encodedFile = jsonSubmission.submission + val jarContent = decodeBase64(encodedFile) + try { + sbt.IO.write(targetJar, jarContent) + QueueResult(jsonSubmission.api_state).successNel + } catch { + case e: IOException => + ("Failed to write jar file to "+ targetJar.getAbsolutePath +"\n"+ e.toString).failNel + } + } + } yield queueResult + } + + def unpackJar(file: File, targetDirectory: File): ValidationNEL[String, Unit] = { + try { + val files = sbt.IO.unzip(file, targetDirectory) + if (files.isEmpty) + ("No files found when unpacking jar file "+ file.getAbsolutePath).failNel + else + ().successNel + } catch { + case e: IOException => + val msg = "Error while unpacking the jar file "+ file.getAbsolutePath +" to "+ targetDirectory.getAbsolutePath +"\n"+ e.toString + if (Settings.offlineMode) { + println("[offline mode] "+ msg) + ().successNel + } else { + msg.failNel + } + } + } + + + /******************************** + * SUBMITTING GRADES + */ + + def submitGrade(feedback: String, score: String, apiState: String, apiKey: String, gradeProject: ProjectDetails): ValidationNEL[String, Unit] = { + import DefaultJsonProtocol._ + val baseReq = url(Settings.uploadFeedbackUrl(gradeProject.courseId)) + val withArgs = baseReq << Map("api_state" -> apiState, "score" -> score, "feedback" -> feedback) <:< Map("X-api-key" -> apiKey) + executeRequest(withArgs) { res => + try { + val js = JsonParser(res) + val status = (js \ "status").convertTo[String] + if (status == "202") + ().successNel + else + ("Unexpected result from submit request: "+ status).failNel + } catch { + case e: Exception => + ("Failed to parse response while submitting grade\n"+ res +"\n"+ fullExceptionString(e)).failNel + } + } + } + + + /********************************* + * TOOLS AND STUFF + */ + + def shaHexDigest(s: String): String = { + val chars = Hex.encodeHex(DigestUtils.sha(s)) + new String(chars) + } + + + def fullExceptionString(e: Throwable) = + e.toString +"\n"+ e.getStackTrace.map(_.toString).mkString("\n") + + + /* Base 64 tools */ + + def encodeBase64(bytes: Array[Byte]): String = + new String(Base64.encodeBase64(bytes)) + + def decodeBase64(str: String): Array[Byte] = { + // codecs 1.4 has a version accepting a string, but not 1.2; jar hell. + Base64.decodeBase64(str.getBytes) + } +} + +case class Challenge(email: String, challengeKey: String, state: String) + +case class QueueResult(apiState: String) + diff --git a/Scala/objsets/project/GradingFeedback.scala b/Scala/objsets/project/GradingFeedback.scala new file mode 100644 index 0000000..5d78c54 --- /dev/null +++ b/Scala/objsets/project/GradingFeedback.scala @@ -0,0 +1,218 @@ +import collection.mutable.ListBuffer +import org.apache.commons.lang3.StringEscapeUtils + +object GradingFeedback { + + private val feedbackSummary = new ListBuffer[String]() + private val feedbackDetails = new ListBuffer[String]() + + private def addSummary(msg: String) { feedbackSummary += msg; feedbackSummary += "\n\n" } + private def addDetails(msg: String) { feedbackDetails += msg; feedbackDetails += "\n\n" } + + /** + * Converts the string to HTML - coursera displays the feedback in an html page. + */ + def feedbackString(html: Boolean = true) = { + val total = totalGradeMessage(totalScore) + "\n\n" + // trim removes the newlines at the end + val s = (total + feedbackSummary.mkString + feedbackDetails.mkString).trim + if (html) + "<pre>"+ StringEscapeUtils.escapeHtml4(s) +"</pre>" + else + s + } + + private var vTestScore: Double = 0d + private var vStyleScore: Double = 0d + def totalScore = vTestScore + vStyleScore + + private var vMaxTestScore: Double = 0d + private var vMaxStyleScore: Double = 0d + def maxTestScore = vMaxTestScore + def maxStyleScore = vMaxStyleScore + + // a string obtained from coursera when downloading an assignment. it has to be + // used again when uploading the grade. + var apiState: String = "" + + /** + * `failed` means that there was an unexpected error during grading. This includes + * - student's code does not compile + * - our tests don't compile (against the student's code) + * - crash while executing ScalaTest (not test failures, but problems trying to run the tests!) + * - crash while executing the style checker (again, not finding style problems!) + * + * When failed is `true`, later grading stages will not be executed: this is handled automatically + * by SBT, tasks depending on a failed one are not run. + * + * However, these dependent tasks still fail (i.e. mapR on them is invoked). The variable below + * allows us to know if something failed before. In this case, we don't add any more things to + * the log. (see `ProgFunBuild.handleFailure`) + */ + private var failed = false + def isFailed = failed + + def initialize() { + feedbackSummary.clear() + feedbackDetails.clear() + vTestScore = 0d + vStyleScore = 0d + apiState = "" + failed = false + } + + def setMaxScore(maxScore: Double, styleScoreRatio: Double) { + vMaxTestScore = maxScore * (1-styleScoreRatio) + vMaxStyleScore = maxScore * styleScoreRatio + } + + + /* Methods to build up the feedback log */ + + def downloadUnpackFailed(log: String) { + failed = true + addSummary(downloadUnpackFailedMessage) + addDetails("======== FAILURES WHILE DOWNLOADING OR EXTRACTING THE SUBMISSION ========") + addDetails(log) + } + + + def compileFailed(log: String) { + failed = true + addSummary(compileFailedMessage) + addDetails("======== COMPILATION FAILURES ========") + addDetails(log) + } + + def testCompileFailed(log: String) { + failed = true + addSummary(testCompileFailedMessage) + addDetails("======== TEST COMPILATION FAILURES ========") + addDetails(log) + } + + + + def allTestsPassed() { + addSummary(allTestsPassedMessage) + vTestScore = maxTestScore + } + + def testsFailed(log: String, score: Double) { + addSummary(testsFailedMessage(score)) + vTestScore = score + addDetails("======== LOG OF FAILED TESTS ========") + addDetails(log) + } + + def testExecutionFailed(log: String) { + failed = true + addSummary(testExecutionFailedMessage) + addDetails("======== ERROR LOG OF TESTING TOOL ========") + addDetails(log) + } + + def testExecutionDebugLog(log: String) { + addDetails("======== DEBUG OUTPUT OF TESTING TOOL ========") + addDetails(log) + } + + + + def perfectStyle() { + addSummary(perfectStyleMessage) + vStyleScore = maxStyleScore + } + + def styleProblems(log: String, score: Double) { + addSummary(styleProblemsMessage(score)) + vStyleScore = score + addDetails("======== CODING STYLE ISSUES ========") + addDetails(log) + } + + + + /* Feedback Messages */ + + private val downloadUnpackFailedMessage = + """We were not able to download your submission from the coursera servers, or extracting the + |archive containing your source code failed. + | + |If you see this error message as your grade feedback, please contact one of the teaching + |assistants. See below for a detailed error log.""".stripMargin + + private val compileFailedMessage = + """We were not able to compile the source code you submitted. This is not expected to happen, + |because the `submit` command in SBT can only be executed if your source code compiles. + | + |Please verify the following points: + | - You should use the `submit` command in SBT to upload your solution + | - You should not perform any changes to the SBT project definition files, i.e. the *.sbt + | files, and the files in the `project/` directory + | + |Take a careful look at the compiler output below - maybe you can find out what the problem is. + | + |If you cannot find a solution, ask for help on the discussion forums on the course website.""".stripMargin + + + private val testCompileFailedMessage = + """We were not able to compile our tests, and therefore we could not correct your submission. + | + |The most likely reason for this problem is that your submitted code uses different names + |for methods, classes, objects or different types than expected. + | + |In principle, this can only arise if you changed some names or types in the code that we + |provide, for instance a method name or a parameter type. + | + |To diagnose your problem, perform the following steps: + | - Run the tests that we provide with our hand-out. These tests verify that all names and + | types are correct. In case these tests pass, but you still see this message, please post + | a report on the forums [1]. + | - Take a careful look at the error messages from the Scala compiler below. They should give + | you a hint where your code has an unexpected shape. + | + |If you cannot find a solution, ask for help on the discussion forums on the course website.""".stripMargin + + + private def testsFailedMessage(score: Double) = + """The code you submitted did not pass all of our tests: your submission achieved a score of + |%.2f out of %.2f in our tests. + | + |In order to find bugs in your code, we advise to perform the following steps: + | - Take a close look at the test output that you can find below: it should point you to + | the part of your code that has bugs. + | - Run the tests that we provide with the handout on your code. + | - The tests we provide do not test your code in depth: they are very incomplete. In order + | to test more aspects of your code, write your own unit tests. + | - Take another very careful look at the assignment description. Try to find out if you + | misunderstood parts of it. While reading through the assignment, write more tests. + | + |Below you can find a short feedback for every individual test that failed.""".stripMargin.format(score, vMaxTestScore) + + // def so that we read the right value of vMaxTestScore (initialize modifies it) + private def allTestsPassedMessage = + """Your solution passed all of our tests, congratulations! You obtained the maximal test + |score of %.2f.""".stripMargin.format(vMaxTestScore) + + private val testExecutionFailedMessage = + """An error occured while running our tests on your submission. This is not expected to + |happen, it means there is a bug in our testing environment. + | + |In order for us to help you, please contact one of the teaching assistants and send + |them the entire feedback message that you recieved.""".stripMargin + + // def so that we read the right value of vMaxStyleScore (initialize modifies it) + private def perfectStyleMessage = + """Our automated style checker tool could not find any issues with your code. You obtained the maximal + |style score of %.2f.""".stripMargin.format(vMaxStyleScore) + + + private def styleProblemsMessage(score: Double) = + """Our automated style checker tool found issues in your code with respect to coding style: it + |computed a style score of %.2f out of %.2f for your submission. See below for detailed feedback.""".stripMargin.format(score, vMaxStyleScore) + + + private def totalGradeMessage(score: Double) = + """Your overall score for this assignment is %.2f out of %.2f""".format(score, vMaxTestScore + vMaxStyleScore) +} diff --git a/Scala/objsets/project/ProgFunBuild.scala b/Scala/objsets/project/ProgFunBuild.scala new file mode 100644 index 0000000..93d4b9d --- /dev/null +++ b/Scala/objsets/project/ProgFunBuild.scala @@ -0,0 +1,646 @@ +import sbt._ +import Keys._ + +import scalaz.Scalaz.mkIdentity +import scalaz.{Success, Failure} +import com.typesafe.sbteclipse.plugin.EclipsePlugin.EclipseKeys + +/** + * See README.md for high-level overview + * + * Libraries Doc Links + * + * Coursera API + * - http://support.coursera.org/customer/portal/articles/573466-programming-assignments + * - the python script 'submit.py' that can be downloaded from the above site + * + * SBT + * - https://github.com/harrah/xsbt/wiki/Getting-Started-Full-Def + * - https://github.com/harrah/xsbt/wiki/Getting-Started-Custom-Settings + * - https://github.com/harrah/xsbt/wiki/Getting-Started-More-About-Settings + * - https://github.com/harrah/xsbt/wiki/Input-Tasks + * - https://github.com/harrah/xsbt/wiki/Tasks + * - http://harrah.github.com/xsbt/latest/api/index.html + * - https://groups.google.com/forum/?fromgroups#!forum/simple-build-tool + * + * Dispatch + * - http://dispatch-classic.databinder.net/Response+Bodies.html + * - http://www.flotsam.nl/dispatch-periodic-table.html + * - http://databinder.net/dispatch-doc/ + * + * Scalaz + * - http://www.lunatech-research.com/archives/2012/03/02/validation-scala + * - http://scalaz.github.com/scalaz/scalaz-2.9.1-6.0.4/doc/index.html#scalaz.Validation + * + * Apache Commons Codec 1.4 + * - http://www.jarvana.com/jarvana/view/commons-codec/commons-codec/1.4/commons-codec-1.4-javadoc.jar!/index.html + * + * Scalatest + * - http://doc.scalatest.org/1.9.1/index.html#org.scalatest.package + */ +object ProgFunBuild extends Build { + + /*********************************************************** + * MAIN PROJECT DEFINITION + */ + + lazy val assignmentProject = Project(id = "assignment", base = file(".")) settings( + // 'submit' depends on 'packageSrc', so needs to be a project-level setting: on build-level, 'packageSrc' is not defined + submitSetting, + createHandoutSetting, + // put all libs in the lib_managed directory, that way we can distribute eclipse project files + retrieveManaged := true, + EclipseKeys.relativizeLibs := true, + // Avoid generating eclipse source entries for the java directories + (unmanagedSourceDirectories in Compile) <<= (scalaSource in Compile)(Seq(_)), + (unmanagedSourceDirectories in Test) <<= (scalaSource in Test)(Seq(_)), + commonSourcePackages := Seq(), // see build.sbt + gradingTestPackages := Seq(), // see build.sbt + selectMainSources, + selectTestSources, + scalaTestSetting, + styleCheckSetting, + setTestPropertiesSetting, + setTestPropertiesHook + ) settings (packageSubmissionFiles: _*) + + + /*********************************************************** + * SETTINGS AND TASKS + */ + + /** The 'submit' task uses this project name (defined in the build.sbt file) to know where to submit the solution */ + val submitProjectName = SettingKey[String]("submitProjectName") + + /** Project-specific settings, see main build.sbt */ + val projectDetailsMap = SettingKey[Map[String, ProjectDetails]]("projectDetailsMap") + + /** + * The files that are handed out to students. Accepts a string denoting the project name for + * which a handout will be generated. + */ + val handoutFiles = TaskKey[String => PathFinder]("handoutFiles") + + /** + * This setting allows to restrict the source files that are compiled and tested + * to one specific project. It should be either the empty string, in which case all + * projects are included, or one of the project names from the projectDetailsMap. + */ + val currentProject = SettingKey[String]("currentProject") + + /** Package names of source packages common for all projects, see comment in build.sbt */ + val commonSourcePackages = SettingKey[Seq[String]]("commonSourcePackages") + + /** Package names of test sources for grading, see comment in build.sbt */ + val gradingTestPackages = SettingKey[Seq[String]]("gradingTestPackages") + + /************************************************************ + * SUBMITTING A SOLUTION TO COURSERA + */ + + val packageSubmission = TaskKey[File]("packageSubmission") + + val packageSubmissionFiles = { + // the packageSrc task uses Defaults.packageSrcMappings, which is defined as concatMappings(resourceMappings, sourceMappings) + // in the packageSubmisson task we only use the sources, not the resources. + inConfig(Compile)(Defaults.packageTaskSettings(packageSubmission, Defaults.sourceMappings)) + } + + /** Task to submit a solution to coursera */ + val submit = InputKey[Unit]("submit") + + lazy val submitSetting = submit <<= inputTask { argTask => + (argTask, compile in Compile, currentProject, (packageSubmission in Compile), submitProjectName, projectDetailsMap, streams) map { (args, _, currentProject, sourcesJar, projectName, detailsMap, s) => + if (currentProject != "") { + val msg = + """The 'currentProject' setting is not empty: '%s' + | + |This error only appears if there are mistakes in the build scripts. Please re-download the assignment + |from the coursera webiste. Make sure that you did not perform any changes to the build files in the + |`project/` directory. If this error persits, ask for help on the course forums.""".format(currentProject).stripMargin +"\n " + s.log.error(msg) + failSubmit() + } else { + lazy val wrongNameMsg = + """Unknown project name: %s + | + |This error only appears if there are mistakes in the build scripts. Please re-download the assignment + |from the coursera webiste. Make sure that you did not perform any changes to the build files in the + |`project/` directory. If this error persits, ask for help on the course forums.""".format(projectName).stripMargin +"\n " + // log strips empty lines at the ond of `msg`. to have one we add "\n " + val details = detailsMap.getOrElse(projectName, {s.log.error(wrongNameMsg); failSubmit()}) + args match { + case email :: otPassword :: Nil => + submitSources(sourcesJar, details, email, otPassword, s.log) + case _ => + val msg = + """No e-mail address and / or submission password provided. The required syntax for `submit` is + | submit <e-mail> <submissionPassword> + | + |The submission password, which is NOT YOUR LOGIN PASSWORD, can be obtained from the assignment page + | https://class.coursera.org/%s/assignment/index""".format(details.courseId).stripMargin +"\n " + s.log.error(msg) + failSubmit() + } + } + } + } + + + def submitSources(sourcesJar: File, submitProject: ProjectDetails, email: String, otPassword: String, logger: Logger) { + import CourseraHttp._ + logger.info("Connecting to coursera. Obtaining challenge...") + val res = for { + challenge <- getChallenge(email, submitProject) + chResponse <- { + logger.info("Computing challenge response...") + challengeResponse(challenge, otPassword).successNel[String] + } + response <- { + logger.info("Submitting solution...") + submitSolution(sourcesJar, submitProject, challenge, chResponse) + } + } yield response + + res match { + case Failure(msgs) => + for (msg <- msgs.list) logger.error(msg) + failSubmit() + case Success(response) => + logger.success("Your code was successfully submitted: "+ response) + } + } + + + def failSubmit(): Nothing = { + sys.error("Submission failed") + } + + + + /*********************************************************** + * CREATE THE HANDOUT ZIP FILE + */ + + val createHandout = InputKey[File]("createHandout") + + // depends on "compile in Test" to make sure everything compiles. also makes sure that + // all dependencies are downloaded, because we pack the .jar files into the handout. + lazy val createHandoutSetting = createHandout <<= inputTask { argTask => + (argTask, currentProject, baseDirectory, handoutFiles, submitProjectName, target, projectDetailsMap, compile in Test) map { (args, currentProject, basedir, filesFinder, submitProject, targetDir, detailsMap, _) => + if (currentProject != "") + sys.error("\nthe 'currentProject' setting in build.sbt needs to be \"\" in order to create a handout") + else args match { + case handoutProjectName :: eclipseDone :: Nil if eclipseDone == "eclipseWasCalled" => + if (handoutProjectName != submitProject) + sys.error("\nThe `submitProjectName` setting in `build.sbt` must match the project name for which a handout is generated\n ") + val files = filesFinder(handoutProjectName).get + val filesWithRelativeNames = files.x_!(relativeTo(basedir)) map { + case (file, name) => (file, handoutProjectName+"/"+name) + } + val targetZip = targetDir / (handoutProjectName +".zip") + IO.zip(filesWithRelativeNames, targetZip) + targetZip + case _ => + val msg =""" + | + |Failed to create handout. Syntax: `createHandout <projectName> <eclipseWasCalled>` + | + |Valid project names are: %s + | + |The argument <eclipseWasCalled> needs to be the string "eclipseWasCalled". This is to remind + |you that you **need** to manually run the `eclipse` command before running `createHandout`. + | """.stripMargin.format(detailsMap.keys.mkString(", ")) + sys.error(msg) + } + } + } + + + /************************************************************ + * LIMITING SOURCES TO CURRENT PROJECT + */ + + def filter(basedir: File, packages: Seq[String]) = new FileFilter { + def accept(file: File) = { + basedir.equals(file) || { + IO.relativize(basedir, file) match { + case Some(str) => + packages exists { pkg => + str.startsWith(pkg) + } + case _ => + sys.error("unexpected test file: "+ file +"\nbase dir: "+ basedir) + } + } + } + } + + def projectFiles(allFiles: Seq[File], basedir: File, projectName: String, globalPackages: Seq[String], detailsMap: Map[String, ProjectDetails]) = { + if (projectName == "") allFiles + else detailsMap.get(projectName) match { + case Some(project) => + val finder = allFiles ** filter(basedir, globalPackages :+ project.packageName) + finder.get + case None => + sys.error("currentProject is set to an invalid name: "+ projectName) + } + } + + /** + * Only include source files of 'currentProject', helpful when preparign a specific assignment. + * Also keeps the source packages in 'commonSourcePackages'. + */ + val selectMainSources = { + (unmanagedSources in Compile) <<= (unmanagedSources in Compile, scalaSource in Compile, projectDetailsMap, currentProject, commonSourcePackages) map { (sources, srcMainScalaDir, detailsMap, projectName, commonSrcs) => + projectFiles(sources, srcMainScalaDir, projectName, commonSrcs, detailsMap) + } + } + + /** + * Only include the test files which are defined in the package of the current project. + * Also keeps test sources in packages listed in 'gradingTestPackages'. + */ + val selectTestSources = { + (unmanagedSources in Test) <<= (unmanagedSources in Test, scalaSource in Test, projectDetailsMap, currentProject, gradingTestPackages) map { (sources, srcTestScalaDir, detailsMap, projectName, gradingSrcs) => + projectFiles(sources, srcTestScalaDir, projectName, gradingSrcs, detailsMap) + } + } + + + /************************************************************ + * PARAMETERS FOR RUNNING THE TESTS + * + * Setting some system properties that are parameters for the GradingSuite test + * suite mixin. This is for running the `test` task in SBT's JVM. When running + * the `scalaTest` task, the ScalaTestRunner creates a new JVM and passes the + * same properties. + */ + + val setTestProperties = TaskKey[Unit]("setTestProperties") + val setTestPropertiesSetting = setTestProperties := { + import scala.util.Properties._ + import Settings._ + setProp(scalaTestIndividualTestTimeoutProperty, individualTestTimeout.toString) + setProp(scalaTestDefaultWeigthProperty, scalaTestDefaultWeigth.toString) + } + + val setTestPropertiesHook = (test in Test) <<= (test in Test).dependsOn(setTestProperties) + + + /************************************************************ + * RUNNING WEIGHTED SCALATEST & STYLE CHECKER ON DEVELOPMENT SOURCES + */ + + def copiedResourceFiles(copied: collection.Seq[(java.io.File, java.io.File)]): List[File] = { + copied collect { + case (from, to) if to.isFile => to + } toList + } + + val scalaTest = TaskKey[Unit]("scalaTest") + val scalaTestSetting = scalaTest <<= + (compile in Compile, + compile in Test, + fullClasspath in Test, + copyResources in Compile, + classDirectory in Test, + baseDirectory, + streams) map { (_, _, classpath, resources, testClasses, basedir, s) => + // we use `map`, so this is only executed if all dependencies succeed. no need to check `GradingFeedback.isFailed` + val logger = s.log + val outfile = basedir / Settings.testResultsFileName + val policyFile = basedir / Settings.policyFileName + val (score, maxScore, feedback, runLog) = ScalaTestRunner.runScalaTest(classpath, testClasses, outfile, policyFile, copiedResourceFiles(resources), logger.error(_)) + logger.info(feedback) + logger.info("Test Score: "+ score +" out of "+ maxScore) + if (!runLog.isEmpty) { + logger.info("Console output of ScalaTest process") + logger.info(runLog) + } + } + + val styleCheck = TaskKey[Unit]("styleCheck") + + /** + * depend on compile to make sure the sources pass the compiler + */ + val styleCheckSetting = styleCheck <<= (compile in Compile, sources in Compile, streams) map { (_, sourceFiles, s) => + val logger = s.log + val (feedback, score) = StyleChecker.assess(sourceFiles) + logger.info(feedback) + logger.info("Style Score: "+ score +" out of "+ StyleChecker.maxResult) + } + + + /************************************************************ + * PROJECT DEFINITION FOR GRADING + */ + + lazy val submissionProject = Project(id = "submission", base = file(Settings.submissionDirName)) settings( + /** settings we take over from the assignment project */ + version <<= (version in assignmentProject), + name <<= (name in assignmentProject), + scalaVersion <<= (scalaVersion in assignmentProject), + scalacOptions <<= (scalacOptions in assignmentProject), + libraryDependencies <<= (libraryDependencies in assignmentProject), + + /** settings specific to the grading project */ + initGradingSetting, + // default value, don't change. see comment on `val partIdOfGradingProject` + partIdOfGradingProject := "", + gradeProjectDetailsSetting, + setMaxScoreSetting, + setMaxScoreHook, + // default value, don't change. see comment on `val apiKey` + apiKey := "", + getSubmissionSetting, + getSubmissionHook, + submissionLoggerSetting, + readCompileLog, + readTestCompileLog, + setTestPropertiesSetting, + setTestPropertiesHook, + resourcesFromAssignment, + selectResourcesForProject, + testSourcesFromAssignment, + selectTestsForProject, + scalaTestSubmissionSetting, + styleCheckSubmissionSetting, + gradeSetting, + EclipseKeys.skipProject := true + ) + + /** + * The assignment part id of the project to be graded. Don't hard code this setting in .sbt or .scala, this + * setting should remain a (command-line) parameter of the `submission/grade` task, defined when invoking sbt. + * See also feedback string in "val gradeProjectDetailsSetting". + */ + val partIdOfGradingProject = SettingKey[String]("partIdOfGradingProject") + + /** + * The api key to access non-public api parts on coursera. This key is secret! It's defined in + * 'submission/settings.sbt', which is not part of the handout. + * + * Default value 'apiKey' to make the handout sbt project work + * - In the handout, apiKey needs to be defined, otherwise the build doesn't compile + * - When correcting, we define 'apiKey' in the 'submission/sectrets.sbt' file + * - The value in the .sbt file will take precedence when correcting (settings in .sbt take + * precedence over those in .scala) + */ + val apiKey = SettingKey[String]("apiKey") + + + /************************************************************ + * GRADING INITIALIZATION + */ + + val initGrading = TaskKey[Unit]("initGrading") + lazy val initGradingSetting = initGrading <<= (clean, sourceDirectory, baseDirectory) map { (_, submissionSrcDir, basedir) => + deleteFiles(submissionSrcDir, basedir) + GradingFeedback.initialize() + RecordingLogger.clear() + } + + def deleteFiles(submissionSrcDir: File, basedir: File) { + // don't delete anything in offline mode, useful for us when hacking testing / stylechecking + if (!Settings.offlineMode){ + IO.delete(submissionSrcDir) + IO.delete(basedir / Settings.submissionJarFileName) + IO.delete(basedir / Settings.testResultsFileName) + } + } + + /** ProjectDetails of the project that we are grading */ + val gradeProjectDetails = TaskKey[ProjectDetails]("gradeProjectDetails") + + // here we depend on `initialize` because we already use the GradingFeedback + lazy val gradeProjectDetailsSetting = gradeProjectDetails <<= (initGrading, partIdOfGradingProject, projectDetailsMap in assignmentProject) map { (_, partId, detailsMap) => + detailsMap.find(_._2.assignmentPartId == partId) match { + case Some((_, details)) => + details + case None => + val validIds = detailsMap.map(_._2.assignmentPartId) + val msgRaw = + """Unknown assignment part id: %s + |Valid part ids are: %s + | + |In order to grade a project, the `partIdOfGradingProject` setting has to be defined. If you are running + |interactively in the sbt console, type `set (partIdOfGradingProject in submissionProject) := "idString"`. + |When running the grading task from the command line, add the above `set` command, e.g. execute + | + | sbt 'set (partIdOfGradingProject in submissionProject) := "idString"' submission/grade""" + val msg = msgRaw.stripMargin.format(partId, validIds.mkString(", ")) + "\n " + GradingFeedback.downloadUnpackFailed(msg) + sys.error(msg) + } + } + + val setMaxScore = TaskKey[Unit]("setMaxScore") + val setMaxScoreSetting = setMaxScore <<= (gradeProjectDetails) map { project => + GradingFeedback.setMaxScore(project.maxScore, project.styleScoreRatio) + } + + // set the maximal score before running compile / test / ... + val setMaxScoreHook = (compile in Compile) <<= (compile in Compile).dependsOn(setMaxScore) + + + /************************************************************ + * DOWNLOADING AND EXTRACTING SUBMISSION + */ + + val getSubmission = TaskKey[Unit]("getSubmission") + val getSubmissionSetting = getSubmission <<= (baseDirectory, scalaSource in Compile) map { (baseDir, scalaSrcDir) => + readAndUnpackSubmission(baseDir, scalaSrcDir) + } + + def readAndUnpackSubmission(baseDir: File, targetSourceDir: File) { + try { + val jsonFile = baseDir / Settings.submissionJsonFileName + val targetJar = baseDir / Settings.submissionJarFileName + val res = for { + queueResult <- { + if (Settings.offlineMode) { + println("[not unpacking from json file]") + QueueResult("").successNel + } else { + CourseraHttp.readJsonFile(jsonFile, targetJar) + } + } + _ <- { + GradingFeedback.apiState = queueResult.apiState + CourseraHttp.unpackJar(targetJar, targetSourceDir) + } + } yield () + + res match { + case Failure(msgs) => + GradingFeedback.downloadUnpackFailed(msgs.list.mkString("\n")) + case _ => + () + } + } catch { + case e: Throwable => + // generate some useful feedback in case something fails + GradingFeedback.downloadUnpackFailed(CourseraHttp.fullExceptionString(e)) + throw e + } + if (GradingFeedback.isFailed) failDownloadUnpack() + } + + // dependsOn makes sure that `getSubmission` is executed *before* `unmanagedSources` + val getSubmissionHook = (unmanagedSources in Compile) <<= (unmanagedSources in Compile).dependsOn(getSubmission) + + def failDownloadUnpack(): Nothing = { + sys.error("Download or Unpack failed") + } + + /************************************************************ + * READING COMPILATION AND TEST COMPILATION LOGS + */ + + + // extraLoggers need to be defined globally. (extraLoggers in Compile) does not work - sbt only + // looks at the global extraLoggers when creating the LogManager. + val submissionLoggerSetting = extraLoggers ~= { currentFunction => + (key: ScopedKey[_]) => { + new FullLogger(RecordingLogger) +: currentFunction(key) + } + } + + val readCompileLog = (compile in Compile) <<= (compile in Compile) mapR handleFailure(compileFailed) + val readTestCompileLog = (compile in Test) <<= (compile in Test) mapR handleFailure(compileTestFailed) + + def handleFailure[R](handler: (Incomplete, String) => Unit) = (res: Result[R]) => res match { + case Inc(inc) => + // Only call the handler of the task that actually failed. See comment in GradingFeedback.failed + if (!GradingFeedback.isFailed) + handler(inc, RecordingLogger.readAndClear()) + throw inc + case Value(v) => v + } + + def compileFailed(inc: Incomplete, log: String) { + GradingFeedback.compileFailed(log) + } + + def compileTestFailed(inc: Incomplete, log: String) { + GradingFeedback.testCompileFailed(log) + } + + + /************************************************************ + * RUNNING SCALATEST + */ + + /** The submission project takes resource files from the main (assignment) project */ + val resourcesFromAssignment = { + (resourceDirectory in Compile) <<= (resourceDirectory in (assignmentProject, Compile)) + } + + /** + * Only include the resource files which are defined in the package of the current project. + */ + val selectResourcesForProject = { + (resources in Compile) <<= (resources in Compile, resourceDirectory in (assignmentProject, Compile), gradeProjectDetails) map { (resources, resourceDir, project) => + val finder = resources ** filter(resourceDir, List(project.packageName)) + finder.get + } + } + + /** The submission project takes test files from the main (assignment) project */ + val testSourcesFromAssignment = { + (sourceDirectory in Test) <<= (sourceDirectory in (assignmentProject, Test)) + } + + /** + * Only include the test files which are defined in the package of the current project. + * Also keeps test sources in packages listed in 'gradingTestPackages' + */ + val selectTestsForProject = { + (unmanagedSources in Test) <<= (unmanagedSources in Test, scalaSource in (assignmentProject, Test), gradingTestPackages in assignmentProject, gradeProjectDetails) map { (sources, testSrcScalaDir, gradingSrcs, project) => + val finder = sources ** filter(testSrcScalaDir, gradingSrcs :+ project.packageName) + finder.get + } + } + + val scalaTestSubmission = TaskKey[Unit]("scalaTestSubmission") + val scalaTestSubmissionSetting = scalaTestSubmission <<= + (compile in Compile, + compile in Test, + fullClasspath in Test, + copyResources in Compile, + classDirectory in Test, + baseDirectory) map { (_, _, classpath, resources, testClasses, basedir) => + // we use `map`, so this is only executed if all dependencies succeed. no need to check `GradingFeedback.isFailed` + val outfile = basedir / Settings.testResultsFileName + val policyFile = basedir / ".." / Settings.policyFileName + ScalaTestRunner.scalaTestGrade(classpath, testClasses, outfile, policyFile, copiedResourceFiles(resources)) + } + + + + /************************************************************ + * STYLE CHECKING + */ + + val styleCheckSubmission = TaskKey[Unit]("styleCheckSubmission") + + /** + * - depend on scalaTestSubmission so that test get executed before style checking. the transitive + * dependencies also ensures that the "sources in Compile" don't have compilation errors + * - using `map` makes this task execute only if all its dependencies succeeded. + */ + val styleCheckSubmissionSetting = styleCheckSubmission <<= (sources in Compile, scalaTestSubmission) map { (sourceFiles, _) => + val (feedback, score) = StyleChecker.assess(sourceFiles) + if (score == StyleChecker.maxResult) { + GradingFeedback.perfectStyle() + } else { + val gradeScore = GradingFeedback.maxStyleScore * score / StyleChecker.maxResult + GradingFeedback.styleProblems(feedback, gradeScore) + } + } + + + + /************************************************************ + * SUBMITTING GRADES TO COURSERA + */ + + val grade = TaskKey[Unit]("grade") + + // mapR: submit the grade / feedback in any case, also on failure + val gradeSetting = grade <<= (scalaTestSubmission, styleCheckSubmission, apiKey, gradeProjectDetails, streams) mapR { (_, _, apiKeyR, projectDetailsR, s) => + val logOpt = s match { + case Value(v) => Some(v.log) + case _ => None + } + logOpt.foreach(_.info(GradingFeedback.feedbackString(html = false))) + apiKeyR match { + case Value(apiKey) if (!apiKey.isEmpty) => + // if build failed early, we did not even get the api key from the submission queue + if (!GradingFeedback.apiState.isEmpty && !Settings.offlineMode) { + val scoreString = "%.2f".format(GradingFeedback.totalScore) + val Value(projectDetails) = projectDetailsR + CourseraHttp.submitGrade(GradingFeedback.feedbackString(), scoreString, GradingFeedback.apiState, apiKey, projectDetails) match { + case Failure(msgs) => + sys.error(msgs.list.mkString("\n")) + case _ => + () + } + } else if(Settings.offlineMode) { + logOpt.foreach(_.info(" \nSettings.offlineMode enabled, not uploading the feedback")) + } else { + sys.error("Could not submit feedback - apiState not initialized") + } + case _ => + sys.error("Could not submit feedback - apiKey not defined: "+ apiKeyR) + } + } +} + +case class ProjectDetails(packageName: String, + assignmentPartId: String, + maxScore: Double, + styleScoreRatio: Double, + courseId: String) diff --git a/Scala/objsets/project/RecordingLogger.scala b/Scala/objsets/project/RecordingLogger.scala new file mode 100644 index 0000000..b886768 --- /dev/null +++ b/Scala/objsets/project/RecordingLogger.scala @@ -0,0 +1,35 @@ +import sbt._ +import collection.mutable.ListBuffer + +/** + * Logger to capture compiler output, test output + */ + +object RecordingLogger extends Logger { + private val buffer = ListBuffer[String]() + + def hasErrors = buffer.nonEmpty + + def readAndClear() = { + val res = buffer.mkString("\n") + buffer.clear() + res + } + + def clear() { + buffer.clear() + } + + def log(level: Level.Value, message: => String) = + if (level == Level.Error) { + buffer += message + } + + // we don't log success here + def success(message: => String) = () + + // invoked when a task throws an exception. invoked late, when the exception is logged, i.e. + // just before returning to the prompt. therefore we do nothing: storing the exception in the + // buffer would happen *after* the `handleFailure` reads the buffer. + def trace(t: => Throwable) = () +} diff --git a/Scala/objsets/project/RichJsValue.scala b/Scala/objsets/project/RichJsValue.scala new file mode 100644 index 0000000..ca9ad94 --- /dev/null +++ b/Scala/objsets/project/RichJsValue.scala @@ -0,0 +1,28 @@ +import cc.spray.json._ + +class RichJsValue(js: JsValue) { + def \ (name: String): JsValue = js match { + case JsObject(fields) => + fields(name) + case _ => + throw new IllegalArgumentException("Cannot select field "+ name +" from non-JsObject "+ js) + } + + def hasFieldNamed(name: String) = js match { + case JsObject(fields) => + fields.contains(name) + case _ => + false + } + + def arrayValues: List[JsValue] = js match { + case JsArray(values) => + values + case _ => + throw new IllegalArgumentException("Trying to select values from non-JsArray"+ js) + } +} + +object RichJsValue { + implicit def enrichJsValue(js: JsValue) = new RichJsValue(js) +} diff --git a/Scala/objsets/project/ScalaTestRunner.scala b/Scala/objsets/project/ScalaTestRunner.scala new file mode 100644 index 0000000..af63495 --- /dev/null +++ b/Scala/objsets/project/ScalaTestRunner.scala @@ -0,0 +1,169 @@ +import sbt._ +import Keys._ +import sys.process.{Process => SysProc, ProcessLogger} +import java.util.concurrent._ +import collection.mutable.ListBuffer + +object ScalaTestRunner { + + class LimitedStringBuffer { + val buf = new ListBuffer[String]() + private var lines = 0 + private var lengthCropped = false + + override def toString() = buf.mkString("\n").trim + + def append(s: String) = + if (lines < Settings.maxOutputLines) { + val shortS = + if (s.length > Settings.maxOutputLineLength) { + if (!lengthCropped) { + val msg = + """WARNING: OUTPUT LINES CROPPED + |Your program generates very long lines on the standard (or error) output. Some of + |the lines have been cropped. + |This should not have an impact on your grade or the grading process; however it is + |bad style to leave `print` statements in production code, so consider removing and + |replacing them by proper tests. + |""".stripMargin + buf.prepend(msg) + lengthCropped = true + } + s.substring(0, Settings.maxOutputLineLength) + } else s + buf.append(shortS) + lines += 1 + } else if (lines == Settings.maxOutputLines) { + val msg = + """WARNING: PROGRAM OUTPUT TOO LONG + |Your program generates massive amounts of data on the standard (or error) output. + |You are probably using `print` statements to debug your code. + |This should not have an impact on your grade or the grading process; however it is + |bad style to leave `print` statements in production code, so consider removing and + |replacing them by proper tests. + |""".stripMargin + buf.prepend(msg) + lines += 1 + } + } + + private def forkProcess(proc: SysProc, timeout: Int) { + val executor = Executors.newSingleThreadExecutor() + val future: Future[Unit] = executor.submit(new Callable[Unit] { + def call { proc.exitValue() } + }) + try { + future.get(timeout, TimeUnit.SECONDS) + } catch { + case to: TimeoutException => + future.cancel(true) + throw to + } finally { + executor.shutdown() + } + } + + private def runPathString(file: File) = file.getAbsolutePath().replace(" ", "\\ ") + + private def extractWeights(s: String, logError: String => Unit) = { + try { + val (nums, rest) = s.span(c => c != '\n') + val List(grade, max) = nums.split(';').toList + (grade.toInt, max.toInt, rest.drop(1)) + } catch { + case e: Throwable => + val msg = "Could not extract weight from grading feedback\n"+ s + logError(msg) + throw e + } + } + + + def runScalaTest(classpath: Classpath, testClasses: File, outfile: File, policyFile: File, resourceFiles: List[File], logError: String => Unit) = { + val classpathString = classpath map { + case Attributed(file) => file.getAbsolutePath() + } mkString(":") + + val testRunpath = runPathString(testClasses) + + val outfileStr = outfile.getAbsolutePath + val policyFileStr = policyFile.getAbsolutePath + val resourceFilesString = resourceFiles.map(_.getAbsolutePath).mkString(":") + // Deleting the file is helpful: it makes reading the file below crash in case ScalaTest doesn't + // run as expected. Problem is, it's hard to detect if ScalaTest ran successfully or not: it + // exits with non-zero if there are failed tests, and also if it crashes... + new java.io.File(outfileStr).delete() + + def prop(name: String, value: String) = "-D"+ name +"="+ value + + // we don't specify "-w packageToTest" - the build file only compiles the tests + // for the current project. so we don't need to do it again here. + val cmd = "java" :: + "-Djava.security.manager" :: + prop("java.security.policy", policyFileStr) :: + prop(Settings.scalaTestReportFileProperty, outfileStr) :: + prop(Settings.scalaTestIndividualTestTimeoutProperty, Settings.individualTestTimeout.toString) :: + prop(Settings.scalaTestReadableFilesProperty, resourceFilesString) :: + prop(Settings.scalaTestDefaultWeigthProperty, Settings.scalaTestDefaultWeigth.toString) :: + "-cp" :: classpathString :: + "org.scalatest.tools.Runner" :: + "-R" :: testRunpath :: + "-C" :: "grading.CourseraReporter" :: + Nil + + // process deadlocks in Runner.PassFailReporter.allTestsPassed on runDoneSemaphore.acquire() when + // something is wrong, e.g. when there's an error.. So we have to run it with a timeout. + + val out = new LimitedStringBuffer() + var p: SysProc = null + try { + p = SysProc(cmd).run(ProcessLogger(out.append(_), out.append(_))) + forkProcess(p, Settings.scalaTestTimeout) + } catch { + case e: TimeoutException => + val msg = "Timeout when running ScalaTest\n"+ out.toString() + logError(msg) + p.destroy() + sys.error(msg) + + case e: Throwable => + val msg = "Error occured while running the ScalaTest command\n"+ e.toString +"\n"+ out.toString() + logError(msg) + p.destroy() + throw e + } + + + val feedbackFileContent = try { + io.Source.fromFile(outfileStr).mkString + } catch { + case e: Throwable => + val msg = "Error occured while reading the output file of ScalaTest\n"+ e.toString +"\n"+ out.toString() + logError(msg) + throw e + } + + val (score, maxScore, feedback) = extractWeights(feedbackFileContent, logError) + val runLog = out.toString() + (score, maxScore, feedback, runLog) + } + + def scalaTestGrade(classpath: Classpath, testClasses: File, outfile: File, policyFile: File, resourceFiles: List[File]) { + val (score, maxScore, feedback, runLog) = runScalaTest(classpath, testClasses, outfile, policyFile, resourceFiles, GradingFeedback.testExecutionFailed) + if (score == maxScore) { + GradingFeedback.allTestsPassed() + } else { + val scaledScore = GradingFeedback.maxTestScore * score / maxScore + GradingFeedback.testsFailed(feedback, scaledScore) + } + + // The output `out` should in principle be empty: the reporter we use writes its results to a file. + // however, `out` contains valuable logs in case scalatest fails. We need to put them into the student + // feedback in order to have a chance of debugging problems. + + if (!runLog.isEmpty) { + GradingFeedback.testExecutionDebugLog(runLog) + } + } +} + diff --git a/Scala/objsets/project/Settings.scala b/Scala/objsets/project/Settings.scala new file mode 100644 index 0000000..c8de201 --- /dev/null +++ b/Scala/objsets/project/Settings.scala @@ -0,0 +1,48 @@ +object Settings { + // when changing this, also look at 'scripts/gradingImpl' and the files in s3/settings + // val courseId = "progfun-2012-001" + + def challengeUrl(courseId: String) = "https://class.coursera.org/"+ courseId +"/assignment/challenge" + + def submitUrl(courseId: String) = "https://class.coursera.org/"+ courseId +"/assignment/submit" + + // def forumUrl(courseId: String) = "https://class.coursera.org/"+ courseId +"/forum/index" + + // def submitQueueUrl(courseId: String) = "https://class.coursera.org/"+ courseId +"/assignment/api/pending_submission" + + def uploadFeedbackUrl(courseId: String) = "https://class.coursera.org/"+ courseId +"/assignment/api/score" + + val maxSubmitFileSize = { + val mb = 1024 * 1024 + 10 * mb + } + + val submissionDirName = "submission" + + val testResultsFileName = "scalaTestLog.txt" + val policyFileName = "allowAllPolicy" + val submissionJsonFileName = "submission.json" + val submissionJarFileName = "submittedSrc.jar" + + // time in seconds that we give scalatest for running + val scalaTestTimeout = 240 + val individualTestTimeout = 30 + + // default weight of each test in a GradingSuite, in case no weight is given + val scalaTestDefaultWeigth = 10 + + // when students leave print statements in their code, they end up in the output of the + // system process running ScalaTest (ScalaTestRunner.scala); we need some limits. + val maxOutputLines = 10*1000 + val maxOutputLineLength = 1000 + + val scalaTestReportFileProperty = "scalatest.reportFile" + val scalaTestIndividualTestTimeoutProperty = "scalatest.individualTestTimeout" + val scalaTestReadableFilesProperty = "scalatest.readableFiles" + val scalaTestDefaultWeigthProperty = "scalatest.defaultWeight" + + // debugging / developping options + + // don't decode json and unpack the submission sources, don't upload feedback + val offlineMode = false +} diff --git a/Scala/objsets/project/StyleChecker.scala b/Scala/objsets/project/StyleChecker.scala new file mode 100644 index 0000000..fbc1cdf --- /dev/null +++ b/Scala/objsets/project/StyleChecker.scala @@ -0,0 +1,77 @@ +import sbt.File +import java.io.ByteArrayOutputStream +import java.io.PrintStream +import org.scalastyle._ + +object StyleChecker { + val maxResult = 100 + + class CustomTextOutput[T <: FileSpec]() extends Output[T] { + private val messageHelper = new MessageHelper(this.getClass().getClassLoader()) + + var fileCount: Int = _ + override def message(m: Message[T]): Unit = m match { + case StartWork() => + case EndWork() => + case StartFile(file) => + print("Checking file " + file + "...") + fileCount = 0 + case EndFile(file) => + if (fileCount == 0) println(" OK!") + case StyleError(file, clazz, key, level, args, line, column, customMessage) => + report(line, column, messageHelper.text(level.name), + findMessage(messageHelper, clazz, key, args, customMessage)) + case StyleException(file, clazz, message, stacktrace, line, column) => + report(line, column, "error", message) + } + + private def report(line: Option[Int], column: Option[Int], level: String, message: String) { + if (fileCount == 0) println("") + fileCount += 1 + println(" " + fileCount + ". " + level + pos(line, column) + ":") + println(" " + message) + } + + private def pos(line: Option[Int], column: Option[Int]): String = line match { + case Some(line) => " at line " + line + (column match { + case Some(column) => " character " + column + case None => "" + }) + case None => "" + } + } + + def score(outputResult: OutputResult) = { + val penalties = outputResult.errors + outputResult.warnings + scala.math.max(maxResult - penalties, 0) + } + + def assess(allSources: Seq[File]): (String, Int) = { + val configFile = new File("project/scalastyle_config.xml").getAbsolutePath + + val sources = allSources.filterNot{ f => + val path = f.getAbsolutePath + path.contains("interpreter") || + path.contains("simulations") || + path.contains("fetchtweets") + } + + val messages = new ScalastyleChecker().checkFiles( + ScalastyleConfiguration.readFromXml(configFile), + Directory.getFiles(sources : _*)) + + val output = new ByteArrayOutputStream() + val outputResult = Console.withOut(new PrintStream(output)) { + new CustomTextOutput().output(messages) + } + + val msg = + output.toString + + "Processed " + outputResult.files + " file(s)\n" + + "Found " + outputResult.errors + " errors\n" + + "Found " + outputResult.warnings + " warnings\n" + + (if (outputResult.errors+outputResult.warnings > 0) "Consult the style guide at https://class.coursera.org/progfun-002/wiki/view?page=GradingPolicy" else "") + + (msg, score(outputResult)) + } +} diff --git a/Scala/objsets/project/build.properties b/Scala/objsets/project/build.properties new file mode 100644 index 0000000..4474a03 --- /dev/null +++ b/Scala/objsets/project/build.properties @@ -0,0 +1 @@ +sbt.version=0.12.1 diff --git a/Scala/objsets/project/buildSettings.sbt b/Scala/objsets/project/buildSettings.sbt new file mode 100644 index 0000000..1ed6540 --- /dev/null +++ b/Scala/objsets/project/buildSettings.sbt @@ -0,0 +1,72 @@ +// needed for custom scalastyle package +resolvers += "namin.github.com/maven-repository" at "http://namin.github.com/maven-repository/" + +resolvers += "Spray Repository" at "http://repo.spray.cc/" + +libraryDependencies += "net.databinder" %% "dispatch-http" % "0.8.8" + + libraryDependencies += "org.scalastyle" % "scalastyle_2.9.1" % "0.1.3-SNAPSHOT" + +libraryDependencies += "cc.spray" %% "spray-json" % "1.1.1" + +// need scalatest also as a build dependency: the build implements a custom reporter +libraryDependencies += "org.scalatest" %% "scalatest" % "1.9.1" + +// dispatch uses commons-codec, in version 1.4, so we can't go for 1.6. +// libraryDependencies += "commons-codec" % "commons-codec" % "1.4" + +libraryDependencies += "org.apache.commons" % "commons-lang3" % "3.1" + +// sbteclipse-plugin uses scalaz-core 6.0.3, so we can't go 6.0.4 +// libraryDependencies += "org.scalaz" %% "scalaz-core" % "6.0.3" + +scalacOptions ++= Seq("-deprecation") + +addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "2.1.0") + +// for dependency-graph plugin +// net.virtualvoid.sbt.graph.Plugin.graphSettings + + +// [info] default:default-3fdafc_2.9.1:0.1-SNAPSHOT +// [info] +-cc.spray:spray-json_2.9.1:1.1.1 +// [info] | +-org.parboiled:parboiled-scala:1.0.2 +// [info] | | +-org.parboiled:parboiled-core:1.0.2 +// [info] | | +-org.scala-lang:scala-library:2.9.1 +// [info] | | +// [info] | +-org.scala-lang:scala-library:2.9.1 +// [info] | +// [info] +-com.typesafe.sbteclipse:sbteclipse-plugin:2.1.0 +// [info] | +-com.typesafe.sbteclipse:sbteclipse-core:2.1.0 +// [info] | +-org.scalaz:scalaz-core_2.9.1:6.0.3 +// [info] | +-org.scala-lang:scala-library:2.9.1 +// [info] | +// [info] +-net.databinder:dispatch-http_2.9.1:0.8.8 +// [info] | +-net.databinder:dispatch-core_2.9.1:0.8.8 +// [info] | | +-org.apache.httpcomponents:httpclient:4.1.3 +// [info] | | | +-commons-codec:commons-codec:1.4 +// [info] | | | +-commons-logging:commons-logging:1.1.1 +// [info] | | | +-org.apache.httpcomponents:httpcore:4.1.4 +// [info] | | | +// [info] | | +-org.scala-lang:scala-library:2.9.1 +// [info] | | +// [info] | +-net.databinder:dispatch-futures_2.9.1:0.8.8 +// [info] | | +-org.scala-lang:scala-library:2.9.1 +// [info] | | +// [info] | +-org.apache.httpcomponents:httpclient:4.1.3 +// [info] | | +-commons-codec:commons-codec:1.4 +// [info] | | +-commons-logging:commons-logging:1.1.1 +// [info] | | +-org.apache.httpcomponents:httpcore:4.1.4 +// [info] | | +// [info] | +-org.scala-lang:scala-library:2.9.1 +// [info] | +// [info] +-org.scala-lang:scala-library:2.9.1 +// [info] +-org.scalastyle:scalastyle_2.9.1:0.1.3-SNAPSHOT +// [info] | +-com.github.scopt:scopt_2.9.1:2.0.0 +// [info] | | +-org.scala-lang:scala-library:2.9.1 +// [info] | | +// [info] | +-org.scalariform:scalariform_2.9.1:0.1.1 +// [info] | +-org.scala-lang:scala-library:2.9.1 +// [info] | +// [info] +-org.scalatest:scalatest_2.9.1:1.8 +// [info] +-org.scala-lang:scala-library:2.9.1 diff --git a/Scala/objsets/project/project/buildPlugins.sbt b/Scala/objsets/project/project/buildPlugins.sbt new file mode 100644 index 0000000..47557f4 --- /dev/null +++ b/Scala/objsets/project/project/buildPlugins.sbt @@ -0,0 +1,2 @@ +// the dependency-graph plugin +// addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.6.0") diff --git a/Scala/objsets/project/scalastyle_config.xml b/Scala/objsets/project/scalastyle_config.xml new file mode 100644 index 0000000..9171ed3 --- /dev/null +++ b/Scala/objsets/project/scalastyle_config.xml @@ -0,0 +1,136 @@ +<scalastyle commentFilter="disabled"> + <name>scalastyle Coursera Configuration</name> + <check level="warning" class="org.scalastyle.file.FileTabChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.file.FileLengthChecker" enabled="true"> + <parameters> + <parameter name="maxFileLength"><![CDATA[800]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.file.HeaderMatchesChecker" enabled="false"> + <parameters> + <parameter name="header"><![CDATA[// Copyright (C) 2011-2012 the original author or authors. +// See the LICENCE.txt file distributed with this work for additional +// information regarding copyright ownership. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License.]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.file.FileLineLengthChecker" enabled="false"> + <parameters> + <parameter name="maxLineLength"><![CDATA[160]]></parameter> + <parameter name="tabSize"><![CDATA[4]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true"> + <parameters> + <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true"> + <parameters> + <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true"> + <parameters> + <parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true"> + <parameters> + <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true"> + <parameters> + <parameter name="maxParameters"><![CDATA[8]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="false"> + <parameters> + <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="false"></check> + <check level="warning" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.file.RegexChecker" enabled="false"> + <parameters> + <parameter name="regex"><![CDATA[println]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.TokenChecker" enabled="false"> + <parameters> + <parameter name="regex"><![CDATA[^isInstanceOf$]]></parameter> + <customMessage>Avoid isInstanceOf.</customMessage> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true"> + <parameters> + <parameter name="maxTypes"><![CDATA[30]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true"> + <parameters> + <parameter name="maximum"><![CDATA[10]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.IfBraceChecker" enabled="false"> + <parameters> + <parameter name="singleLineAllowed"><![CDATA[true]]></parameter> + <parameter name="doubleLineAllowed"><![CDATA[false]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true"> + <parameters> + <parameter name="maxLength"><![CDATA[50]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="false"> + <parameters> + <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true"> + <parameters> + <parameter name="regex"><![CDATA[^[^A-Z].*$]]></parameter> + <customMessage>Method name should not start with an upper case letter.</customMessage> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true"> + <parameters> + <parameter name="maxMethods"><![CDATA[30]]></parameter> + </parameters> + </check> + <check level="warning" class="org.scalastyle.scalariform.VarFieldChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.VarLocalChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.WhileChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.RedundantIfChecker" enabled="true"></check> + <check level="warning" class="org.scalastyle.scalariform.TokenChecker" enabled="true"> + <parameters> + <parameter name="regex"><![CDATA[^(ArrayList|ArrayBuffer|mutable)$]]></parameter> + <customMessage>Avoid using mutable collections.</customMessage> + </parameters> + </check> +</scalastyle>
\ No newline at end of file diff --git a/Scala/objsets/src/main/scala/common/package.scala b/Scala/objsets/src/main/scala/common/package.scala new file mode 100644 index 0000000..f1c74c3 --- /dev/null +++ b/Scala/objsets/src/main/scala/common/package.scala @@ -0,0 +1,40 @@ +import java.io.File + +package object common { + + /** An alias for the `Nothing` type. + * Denotes that the type should be filled in. + */ + type ??? = Nothing + + /** An alias for the `Any` type. + * Denotes that the type should be filled in. + */ + type *** = Any + + + /** + * Get a child of a file. For example, + * + * subFile(homeDir, "b", "c") + * + * corresponds to ~/b/c + */ + def subFile(file: File, children: String*) = { + children.foldLeft(file)((file, child) => new File(file, child)) + } + + /** + * Get a resource from the `src/main/resources` directory. Eclipse does not copy + * resources to the output directory, then the class loader cannot find them. + */ + def resourceAsStreamFromSrc(resourcePath: List[String]): Option[java.io.InputStream] = { + val classesDir = new File(getClass.getResource(".").toURI) + val projectDir = classesDir.getParentFile.getParentFile.getParentFile.getParentFile + val resourceFile = subFile(projectDir, ("src" :: "main" :: "resources" :: resourcePath): _*) + if (resourceFile.exists) + Some(new java.io.FileInputStream(resourceFile)) + else + None + } +} diff --git a/Scala/objsets/src/main/scala/objsets/TweetData.scala b/Scala/objsets/src/main/scala/objsets/TweetData.scala new file mode 100644 index 0000000..8af9e68 --- /dev/null +++ b/Scala/objsets/src/main/scala/objsets/TweetData.scala @@ -0,0 +1,719 @@ +package objsets + +// real tweet data, collected on Oct 1 +object TweetData { + val gizmodo = """[ +{ "user": "gizmodo", "text": "Kindle Paperwhite Review: Forget Everything Else, This Is the E-Reader You Want http://t.co/737W6aNC", "retweets": 51.0 }, +{ "user": "gizmodo", "text": "These new Apple patents give a sneak peek at what future iPhone cameras might have in store. http://t.co/0YT9rjxp", "retweets": 49.0 }, +{ "user": "gizmodo", "text": "Ever wonder why the sky is dark at night? Here's your answer. http://t.co/eTKxkcaE", "retweets": 86.0 }, +{ "user": "gizmodo", "text": "The head of Homeland Security stays secure by just not using email, at all. http://t.co/W6KAFEUu", "retweets": 37.0 }, +{ "user": "gizmodo", "text": "This is how graphene will grow the flexible semiconductors of the future: http://t.co/IoEvuxp4", "retweets": 43.0 }, +{ "user": "gizmodo", "text": "It's the tech-based reality TV show you never knew you didn't want: http://t.co/j9J8gAo8", "retweets": 19.0 }, +{ "user": "gizmodo", "text": "How do you make your Steve Jobs sculpture stand out? Easy, mix in some trash you stole from him. http://t.co/mvHBj3CH", "retweets": 15.0 }, +{ "user": "gizmodo", "text": "This awesome baggage roller coaster will make you wish you were a suitcase. http://t.co/ECaE2hgd", "retweets": 17.0 }, +{ "user": "gizmodo", "text": "This price cut is putting the Kindle Paperwhite and the Nook with GlowLight head to head. Fight! http://t.co/1x6nOJGY", "retweets": 15.0 }, +{ "user": "gizmodo", "text": "18 unlucky people who already broke the iPhone 5: http://t.co/9RpvX4te", "retweets": 79.0 }, +{ "user": "gizmodo", "text": "Here's how to ditch cash and start paying for everything electronically. http://t.co/tgrZ50XM", "retweets": 30.0 }, +{ "user": "gizmodo", "text": "If you're going to get a Nexus 7, you should just go out and buy it right now. http://t.co/tACY1YYG", "retweets": 33.0 }, +{ "user": "gizmodo", "text": "The one thing you can buy that will make your smartphone better: http://t.co/rlA32n4r", "retweets": 74.0 }, +{ "user": "gizmodo", "text": "Cold brew Irish coffee! It's the best part of waking up (drunk) http://t.co/cr3oCPKf", "retweets": 23.0 }, +{ "user": "gizmodo", "text": "Maybe the best reminder yet that the coffee shop is not your office: http://t.co/T8aNd3SX", "retweets": 47.0 }, +{ "user": "gizmodo", "text": "Why a brand new, billion-dollar battleship still needs old-timey wooden ladders: http://t.co/75yOeche", "retweets": 13.0 }, +{ "user": "gizmodo", "text": "Time for another story bundle! Seven crime novels, pay what you want, support indie authors! http://t.co/kI7tFMvM", "retweets": 14.0 }, +{ "user": "gizmodo", "text": "Apple's added a Maps section to its App Store to help you find an Apple Maps replacement: http://t.co/rpP0tCMI", "retweets": 114.0 }, +{ "user": "gizmodo", "text": "Man jailed for accidentally sexting his whole address book (including some tweens) http://t.co/5IxaTHkz #WTFriday", "retweets": 65.0 }, +{ "user": "gizmodo", "text": "An Iranian news agency thought an Onion article was real--and plagiarized it: http://t.co/C4lWe1Ij", "retweets": 112.0 }, +{ "user": "gizmodo", "text": "Yep, the TSA is definitely stealing iPads: http://t.co/THYd8MKe", "retweets": 79.0 }, +{ "user": "gizmodo", "text": "A week with the iPhone 5: http://t.co/ReuK1aJs", "retweets": 111.0 }, +{ "user": "gizmodo", "text": "Tim Cook apologizes for Apple Maps and suggests you download... Bing: http://t.co/imQFFOyW", "retweets": 280.0 }, +{ "user": "gizmodo", "text": "Neil Young unveils his new Pono music player: http://t.co/EEMznUio", "retweets": 27.0 }, +{ "user": "gizmodo", "text": "This $50 stick turns any HDTV into an Android-powered smart TV: http://t.co/8FpZUnIE", "retweets": 101.0 }, +{ "user": "gizmodo", "text": "NASA has found proof of a river on Mars: http://t.co/QPPUSQXp", "retweets": 242.0 }, +{ "user": "gizmodo", "text": "14 people who should be ashamed of their @foursquare mayorship http://t.co/7AEHQJLT", "retweets": 50.0 }, +{ "user": "gizmodo", "text": "This team of quadrocopter drones can throw and catch better than you http://t.co/GYdhzZMq", "retweets": 32.0 }, +{ "user": "gizmodo", "text": "Steve Jobs' reality distortion field lives on http://t.co/CDnZi4z8", "retweets": 56.0 }, +{ "user": "gizmodo", "text": "What's the worst children's toy you've ever seen? Let's bask in the awful http://t.co/fk41tJj0", "retweets": 18.0 }, +{ "user": "gizmodo", "text": "Spies like us: why we're all Big Brother now http://t.co/MB3gXB0b", "retweets": 8.0 }, +{ "user": "gizmodo", "text": "18 places carbon fiber just doesn't belong: http://t.co/vJo1Yhlj", "retweets": 21.0 }, +{ "user": "gizmodo", "text": "A woman had a new ear grown on her arm and attached to her head and it is PRETTY GROSS: http://t.co/NyuikEmP", "retweets": 88.0 }, +{ "user": "gizmodo", "text": "That iron man found by Nazis? It's of extraterrestrial origin. Yes, really. http://t.co/hRR7oufa", "retweets": 92.0 }, +{ "user": "gizmodo", "text": "The new best desk toy since... maybe ever? http://t.co/IwuHYYgI", "retweets": 64.0 }, +{ "user": "gizmodo", "text": "Poop snow on ancient burial ground. This can't end well. http://t.co/DAmSgIiu", "retweets": 30.0 }, +{ "user": "gizmodo", "text": "Giant electronic circuits make wonderfully geeky art: http://t.co/um99UW4Y", "retweets": 20.0 }, +{ "user": "gizmodo", "text": "This little robot will help make all that wine you drink: http://t.co/a6W5wY0p", "retweets": 16.0 }, +{ "user": "gizmodo", "text": "Facebook's finally deleting thousands of face accounts: http://t.co/3quTouWv", "retweets": 51.0 }, +{ "user": "gizmodo", "text": "These people actually really, really love Apple Maps: http://t.co/VvWsa5V7", "retweets": 29.0 }, +{ "user": "gizmodo", "text": "Is your iPhone 5 camera seeing purple? Like, where it shouldn't be? http://t.co/EBnaMfFR", "retweets": 37.0 }, +{ "user": "gizmodo", "text": "Ooh, a galaxy-shooting camera you might actually afford http://t.co/VLXkarGV", "retweets": 22.0 }, +{ "user": "gizmodo", "text": "Is this the most stunning shuttle flyover ever? http://t.co/IYKbjmnx", "retweets": 59.0 }, +{ "user": "gizmodo", "text": "Why Apple really ditched Google Maps http://t.co/evVBDYCu", "retweets": 104.0 }, +{ "user": "gizmodo", "text": "Why is this part of Facebook broken? http://t.co/Y9Ur2U5T", "retweets": 12.0 }, +{ "user": "gizmodo", "text": "Don't buy cheap iPhone 5 cables because they don't actually exist yet http://t.co/3LHLeCdO", "retweets": 56.0 }, +{ "user": "gizmodo", "text": "How vicious spyware contaminated hundreds of thousands of rental PCs: http://t.co/MlrUDWZe", "retweets": 12.0 }, +{ "user": "gizmodo", "text": "How Jack Daniels hand crafts every barrel for aging http://t.co/QeYKZDNv", "retweets": 54.0 }, +{ "user": "gizmodo", "text": "Is putting pictures of missing children on 404 pages effective? http://t.co/WFh9tiZU", "retweets": 26.0 }, +{ "user": "gizmodo", "text": "The gigantic crane that builds even more gigantic aircraft carriers http://t.co/KraKD1sE", "retweets": 17.0 }, +{ "user": "gizmodo", "text": "The 20 most obvious PINs are painfully obvious http://t.co/Du9BYjeR", "retweets": 38.0 }, +{ "user": "gizmodo", "text": "This is the closest, clearest view of Mars yet http://t.co/VrbZRVgR", "retweets": 45.0 }, +{ "user": "gizmodo", "text": "Meet the new Apple, where things don't just work http://t.co/XS36kHWi", "retweets": 146.0 }, +{ "user": "gizmodo", "text": "Cheap goodies on Google Play for the next five days: http://t.co/dT3bRBzZ", "retweets": 52.0 }, +{ "user": "gizmodo", "text": "Do phones need to come with chargers these days? http://t.co/AZMsPUeD", "retweets": 21.0 }, +{ "user": "gizmodo", "text": "The font on your car's dash might increase your risk of crashing: http://t.co/R6e6QiWp", "retweets": 27.0 }, +{ "user": "gizmodo", "text": "Holy crap, self-driving cars are now legal in California! http://t.co/nSOwBSZ1", "retweets": 277.0 }, +{ "user": "gizmodo", "text": "Judging by this photo, joining the USAF makes you a superhero http://t.co/XlCb7nK8", "retweets": 16.0 }, +{ "user": "gizmodo", "text": "Warning: Security bug can wipe out your Android phone just by visiting a web page-not only limited to Samsung http://t.co/0y6vnOKw", "retweets": 290.0 }, +{ "user": "gizmodo", "text": "Nikon D600 review: Images this spectacular have never been so cheap http://t.co/PK4LCOwX", "retweets": 33.0 }, +{ "user": "gizmodo", "text": "Oh good, Iran's homemade war drone is in the air http://t.co/ySXchFfv", "retweets": 40.0 }, +{ "user": "gizmodo", "text": "BTW, this is @brentrose tweeting for Gizmodo this week. Follow for witticisms and euphemisms.", "retweets": 0.0 }, +{ "user": "gizmodo", "text": "Major Samsung security bug can wipe your Galaxy phone (updating) http://t.co/n5yDZ3dh", "retweets": 120.0 }, +{ "user": "gizmodo", "text": "Aboard the ligher, smarter, deadlier aircraft carrier of tomorrow http://t.co/xVjQAr6J", "retweets": 22.0 }, +{ "user": "gizmodo", "text": "NASA publishes the most detailed photo of the universe ever captured, and it's stunning http://t.co/68pDKByp", "retweets": 192.0 }, +{ "user": "gizmodo", "text": "How NASA prevents a space plague outbreak http://t.co/jIeI7hPX", "retweets": 20.0 }, +{ "user": "gizmodo", "text": "iPhone 5's brain dissected. Guess what, it's made by Samsung. http://t.co/wSyjvpDc", "retweets": 321.0 }, +{ "user": "gizmodo", "text": "BlackBerry 10 adds so many features that so few people want http://t.co/eaQM0vlS", "retweets": 58.0 }, +{ "user": "gizmodo", "text": "The supercomputer that houses the entire universe http://t.co/KQciouL2", "retweets": 47.0 }, +{ "user": "gizmodo", "text": "Gambling website refunds everyone's Packers bets because of horrible refs http://t.co/bwmgYBDq", "retweets": 121.0 }, +{ "user": "gizmodo", "text": "12 deadly inventions that killed their creators (not for the faint of heart) http://t.co/BCwRAzhe", "retweets": 82.0 }, +{ "user": "gizmodo", "text": "You won't believe this amazing this wildlife photography contest winner http://t.co/nHIRgFVG", "retweets": 20.0 }, +{ "user": "gizmodo", "text": "How to build an iPhone 5 dock for $1.27 http://t.co/kqsQ1GIV", "retweets": 65.0 }, +{ "user": "gizmodo", "text": "Would you eat this disgusting stuff? (Spoiler: yes, you would) http://t.co/mRAmDRpk", "retweets": 25.0 }, +{ "user": "gizmodo", "text": "The iPhone 5 'shortage' is apparently a result of its new ultra-thin display: http://t.co/RqUSuYif", "retweets": 48.0 }, +{ "user": "gizmodo", "text": "How much do you care about how Twitter measures your influence? http://t.co/0fitjfCD", "retweets": 19.0 }, +{ "user": "gizmodo", "text": "Eric Schmidt confirms a Google Maps app on iOS 6 is still some way off: http://t.co/bobRuY06", "retweets": 76.0 }, +{ "user": "gizmodo", "text": "Breville YouBrew coffee maker review: excellent, expensive, safe for idiots (like @harrysawyers) http://t.co/67wKNuLM", "retweets": 11.0 }, +{ "user": "gizmodo", "text": "Can this flashy redesign make Myspace cool again? http://t.co/OfHVPHz6", "retweets": 94.0 }, +{ "user": "gizmodo", "text": "No, your private Facebook messages have not gone public. Filthy secrets, still safe. http://t.co/RAJsAyK0", "retweets": 67.0 }, +{ "user": "gizmodo", "text": "iOS 5.1 for Apple TV brings new AirPlay goodness http://t.co/1Yj55T52", "retweets": 48.0 }, +{ "user": "gizmodo", "text": "The Facebook for poor people you didn't know existed http://t.co/9zmMTZci", "retweets": 29.0 }, +{ "user": "gizmodo", "text": "(In case you're wondering who the awesome speller is, that's @brentrose. He has an MFA.)", "retweets": 2.0 }, +{ "user": "gizmodo", "text": "iPhone 5 vs Galaxy S III: Who's screen is prettier? http://t.co/n6CbaspY", "retweets": 108.0 }, +{ "user": "gizmodo", "text": "Yeah, we live in the future, but there's all this awesome 50-year-old Jetsons tech we still don't have http://t.co/lXsqu49Z", "retweets": 46.0 }, +{ "user": "gizmodo", "text": "This tiny telescope implant gives eyesight to the blind http://t.co/W50RHP4L", "retweets": 40.0 }, +{ "user": "gizmodo", "text": "Iran shuts down Google, will completely cut citizens fff the internet http://t.co/60ZucBic", "retweets": 146.0 }, +{ "user": "gizmodo", "text": "10 stupid, crazy, wonderful gadget fails http://t.co/p3Al28G5", "retweets": 32.0 }, +{ "user": "gizmodo", "text": "Watch an NBA player dunk the iPhone 5. Boomshakalaka! http://t.co/UQAX0awf", "retweets": 47.0 }, +{ "user": "gizmodo", "text": "Is your iPhone 5... rattling? http://t.co/mn0r2dhb", "retweets": 67.0 }, +{ "user": "gizmodo", "text": "The definitive comparison of iOS 5 Google Maps vs iOS 6 Apple Maps in one single image: http://t.co/fTwTfVMy", "retweets": 191.0 }, +{ "user": "gizmodo", "text": "Study splits breast cancer into four treatable types: http://t.co/myTn3LHu", "retweets": 23.0 }, +{ "user": "gizmodo", "text": "IBM's supercomputer genius Watson is headed for the cloud: http://t.co/Z1psttXB", "retweets": 59.0 }, +{ "user": "gizmodo", "text": "Nuance is planning to develop voice recognition which works while your phone sleeps. Good idea? http://t.co/WEXD9bQa", "retweets": 25.0 }, +{ "user": "gizmodo", "text": "Is your new iPhone picking up more scratches than you'd like? http://t.co/DGEiawOi", "retweets": 35.0 }, +{ "user": "gizmodo", "text": "Foxconn has shut down a factory after 2,000 of its employees started a massive brawl. http://t.co/d7TXo3K9", "retweets": 90.0 }, +{ "user": "gizmodo", "text": "If you thought the NES Zapper was cool before, watch it set things on fire with a laser. http://t.co/t3f0oE8O", "retweets": 37.0 }, +{ "user": "gizmodo", "text": "Want to hack NFC to get free train rides? There's an app for that. http://t.co/eAp6yTbE", "retweets": 82.0 }, +{ "user": "gizmodo", "text": "The weirdest thing people hate about the iPhone 5: http://t.co/GMwuRp8D", "retweets": 202.0 }, +{ "user": "gizmodo", "text": "Data centers waste a completely absurd amount of energy. http://t.co/NcO9pXqb", "retweets": 58.0 } +]""" + val TechCrunch = """[ +{ "user": "TechCrunch", "text": "Resignation Media Hires CEO John Ellis To Run Tapiture, Its Fast-Growing Pinterest For Men http://t.co/ctn7oWJc by @anthonyha", "retweets": 18.0 }, +{ "user": "TechCrunch", "text": "FreedomPop Opens Its Freemium Internet Service To The Masses With New Public Beta http://t.co/35mA9Adp by @chrisvelazco", "retweets": 27.0 }, +{ "user": "TechCrunch", "text": "Dish And The Dream Of Internet TV http://t.co/y8KcSl8G by @ryanlawler", "retweets": 25.0 }, +{ "user": "TechCrunch", "text": "Adobe's Acrobat XI Boasts New PDF Editor And Touch-Friendly Interface ? Upgrades Start At $139 http://t.co/1YDWvlVI by @anthonyha", "retweets": 26.0 }, +{ "user": "TechCrunch", "text": "Testing Out Bodymetrics, The Startup That Wants To Be A Denim Shopper's Best Friend [TCTV] http://t.co/sPe6wA02 by @loyalelectron", "retweets": 22.0 }, +{ "user": "TechCrunch", "text": "Up Close With The Next Big Home Commodity: LED Lighting http://t.co/nGPSMnMH", "retweets": 17.0 }, +{ "user": "TechCrunch", "text": "Cloning Instagram For Video Will Not Revolutionize Mobile Video http://t.co/B5DMkSnQ by @sandeepcasi", "retweets": 43.0 }, +{ "user": "TechCrunch", "text": "(R)evolution http://t.co/dzQFqjBh by @sarahintampa", "retweets": 20.0 }, +{ "user": "TechCrunch", "text": "An Analysis Of Market Demand For Web Programming Languages http://t.co/gdYrXz7i by @marcgayle", "retweets": 91.0 }, +{ "user": "TechCrunch", "text": "The Kindle Paperwhite Is A Reader's Dream http://t.co/jGslGazO by @johnbiggs", "retweets": 61.0 }, +{ "user": "TechCrunch", "text": "Soon-To-Be-Acquired BlueSprig's AirCover Family Locator Is An iOS/Android App That Lets You Track ... http://t.co/qSQquuLS by @ingridlunden", "retweets": 24.0 }, +{ "user": "TechCrunch", "text": "Data Markets: The Emerging Data Economy http://t.co/lnTekycH by @gilelbaz", "retweets": 63.0 }, +{ "user": "TechCrunch", "text": "CoCoon, The Newest Home For Startups In Hong Kong http://t.co/IN164KNl by @mulligan", "retweets": 35.0 }, +{ "user": "TechCrunch", "text": "Imagine No Ads On Facebook. It's Easy If You Try http://t.co/kWSYY2v8 by @joshconstine", "retweets": 70.0 }, +{ "user": "TechCrunch", "text": "Microsoft Needs Windows Phone 7 - Not WP8 - To Win Significant Mobile Market Share http://t.co/X71YJ0MD by @riptari", "retweets": 36.0 }, +{ "user": "TechCrunch", "text": "Open Source Fear Mongering Is Ridiculous With The Advent Of Open APIs http://t.co/evVLkzCu by @alexwilliams", "retweets": 41.0 }, +{ "user": "TechCrunch", "text": "Logitech UE Boombox And Mobile Boombox Review: Bluetooth Speakers With A Rich Sound http://t.co/ZpewcBEX by @romaindillet", "retweets": 17.0 }, +{ "user": "TechCrunch", "text": "You Don't Need A Prototype To Raise A Seed Round http://t.co/vkPaK3sM by @VCMike", "retweets": 59.0 }, +{ "user": "TechCrunch", "text": "Jason Calacanis' Next Act, And Another Pivot For http://t.co/XBhz5HpF, As A 'Knowledge Community' http://t.co/7ZLTckm6 by @ingridlunden", "retweets": 12.0 }, +{ "user": "TechCrunch", "text": "Mass Persuasion, One User at a Time http://t.co/0Cv9fd4V by @nireyal", "retweets": 24.0 }, +{ "user": "TechCrunch", "text": "The Zooka Wireless Speaker Bar Turns Bad Audio Into Loud Noises! http://t.co/VNljouTp by @jordanrcrook", "retweets": 34.0 }, +{ "user": "TechCrunch", "text": "Iterations: We Know About B2B And B2C, But Don't Overlook B2D http://t.co/i0FIyDbX by @semil", "retweets": 55.0 }, +{ "user": "TechCrunch", "text": "5 Big Map App Issues Apple Must Solve http://t.co/XGV7PuXk", "retweets": 82.0 }, +{ "user": "TechCrunch", "text": "B&N Drops Price Of Its Nook GlowLight In Advance Of Amazon's New Reader http://t.co/TkhEJpoz by @johnbiggs", "retweets": 26.0 }, +{ "user": "TechCrunch", "text": "Why Angel Investors Don?t Make Money ? And Advice For People Who Are Going To Become Angels Anyway http://t.co/u2ApCrw7", "retweets": 138.0 }, +{ "user": "TechCrunch", "text": "Barnes & Noble Cuts GlowLight Nook Price To $129 As Amazon Prepares To Ship Its Own Backlit Kindles http://t.co/kiDTncdF by @chrisvelazco", "retweets": 40.0 }, +{ "user": "TechCrunch", "text": "Games Are A Difficult Investment Proposition, But Crowdfunding Could Change That http://t.co/EvpdbrsS", "retweets": 43.0 }, +{ "user": "TechCrunch", "text": "Simon Cowell And http://t.co/7lk1PBvp Are Creating An X Factor To Find The Next Mark Zuckerberg http://t.co/ZesTDWMs by @leenarao", "retweets": 192.0 }, +{ "user": "TechCrunch", "text": "How Technology Is Empowering Teachers, Minting Millionaires, And Improving Education http://t.co/EgFaGI6w by @ripemp", "retweets": 154.0 }, +{ "user": "TechCrunch", "text": "Italians Take Up The Torch To Ignite Their Own Tech Startup Scene http://t.co/zAuoFO4t by @mikebutcher", "retweets": 89.0 }, +{ "user": "TechCrunch", "text": "A Tech Way Around \"Creative Block\" http://t.co/H6eOyiay by @alexcornell", "retweets": 50.0 }, +{ "user": "TechCrunch", "text": "Klout Would Like Potential Employers To Consider Your Score Before Hiring You. And That's Stupid. http://t.co/YmnMj271 by @thatdrew", "retweets": 204.0 }, +{ "user": "TechCrunch", "text": "A Venture Capitalist's E-Commerce Shopping List http://t.co/tA7VbXBT by @VCSerge", "retweets": 55.0 }, +{ "user": "TechCrunch", "text": "Five Big Changes In The iOS 6 App Store (And What Developers Should Do) http://t.co/pkMSN96G by @sarahintampa", "retweets": 76.0 }, +{ "user": "TechCrunch", "text": "Canadian Internet Provider Rogers Experiencing Major, Prolonged Outage http://t.co/xIKbMsou by @drizzled", "retweets": 134.0 }, +{ "user": "TechCrunch", "text": "Hate http://t.co/y6Udo2nZ", "retweets": 38.0 }, +{ "user": "TechCrunch", "text": "The Search For Minority Entrepreneurs Is Over -- Now They Need To Be Ready For Investors http://t.co/8PE4Jjp3 by @waynesutton", "retweets": 59.0 }, +{ "user": "TechCrunch", "text": "Kickstarter: Helios, An iPhone Telepresence Rig On A Budget http://t.co/EgEKtsvt by @johnbiggs", "retweets": 33.0 }, +{ "user": "TechCrunch", "text": "Facebook Updates Messenger For iOS With New Chat UI, iOS 6 And iPhone 5 Support http://t.co/Ffc2a6Ib by @fredericl", "retweets": 66.0 }, +{ "user": "TechCrunch", "text": "The Next Big E-Commerce Wave: Vertically Integrated Commerce http://t.co/MEND0OQk by @bwertz", "retweets": 132.0 }, +{ "user": "TechCrunch", "text": "Bootstrap's Maintainers Leave Twitter For GitHub And Obvious Corp., Will Move It Into Its Own Organiz... http://t.co/jznaaxc1 by @fredericl", "retweets": 76.0 }, +{ "user": "TechCrunch", "text": "Should You Trust Your Gut? The Answer Is Yes. http://t.co/8TlMTip1", "retweets": 74.0 }, +{ "user": "TechCrunch", "text": "PSA For Win8 Devs: The Only Way To Distribute Your Metro Apps Is Through The Windows Store http://t.co/v26GTcsO by @fredericl", "retweets": 22.0 }, +{ "user": "TechCrunch", "text": "Gillmor Gang: Platformicide http://t.co/Sk9zwRw4 by @stevegillmor", "retweets": 17.0 }, +{ "user": "TechCrunch", "text": "Apple's Maps Is A Black Eye, Nothing More http://t.co/XK9H5B5h by @jdalrymple", "retweets": 71.0 }, +{ "user": "TechCrunch", "text": "Lessons From The Dramatic Slow-Motion Death Of Wikitravel http://t.co/u7BUYnwc by @rezendi", "retweets": 43.0 }, +{ "user": "TechCrunch", "text": "Startup Success Requires The Drive For Data http://t.co/Yl2JpFCI by @jeffma", "retweets": 78.0 }, +{ "user": "TechCrunch", "text": "Here Are The Singers Competing In Next Week's 'American Idol For The Geek Set' http://t.co/OcBvSOLl by @anthonyha", "retweets": 20.0 }, +{ "user": "TechCrunch", "text": "Ford CTO Paul Mascarenas On Bridging The Worlds Of Silicon Valley And Motor City [TCTV] http://t.co/oeD4AD5n by @loyalelectron", "retweets": 21.0 }, +{ "user": "TechCrunch", "text": "Apple Adds A Clarifying Description To Its \"Apps For Passbook\" Page In The App Store http://t.co/fkj4P0EP by @drizzled", "retweets": 42.0 }, +{ "user": "TechCrunch", "text": "Post-SingTel Acquisition, Photo Aggregation App Pixable Gets An Image-Centric Redesign http://t.co/pNS4tFke by @anthonyha", "retweets": 12.0 }, +{ "user": "TechCrunch", "text": "Rest In Peace, Charles Alfred Eldon: A Pioneer Of Silicon Valley, A Role Model For This New Generation http://t.co/3p1mt486 by @eldon", "retweets": 89.0 }, +{ "user": "TechCrunch", "text": "TechCrunch Giveaway: Free Tickets To Box's 2012 #BoxWorks Event http://t.co/t4l8IIrK by @elinblesener", "retweets": 35.0 }, +{ "user": "TechCrunch", "text": "Cloud Security Firm Qualys' IPO Opens At $12/Share, Raising $71.8M http://t.co/n2WNQkuH by @ingridlunden", "retweets": 18.0 }, +{ "user": "TechCrunch", "text": "The FeedBurner Deathwatch Continues: Google Kills AdSense For Feeds http://t.co/CztoKUFP by @fredericl", "retweets": 31.0 }, +{ "user": "TechCrunch", "text": "Google's \"Spring\" Cleaning In Fall: AdSense For Feeds, Classic Plus, Spreadsheet Gadgets Get The Axe http://t.co/pSeE3fCS by @thatdrew", "retweets": 26.0 }, +{ "user": "TechCrunch", "text": "Gillmor Gang Live 09.28.12 (TCTV) http://t.co/eUf0Df0q by @stevegillmor", "retweets": 6.0 }, +{ "user": "TechCrunch", "text": "Facebook Shares Jump More Than 6% After Gifts Launch. (Hooray For New Revenue Streams.) http://t.co/8fEXj2kG by @kimmaicutler", "retweets": 99.0 }, +{ "user": "TechCrunch", "text": "Evernote Listens To Unhappy Skitch 2.0 Users: Brings Back Some Old Features And Version 1.0 http://t.co/y2YRxxyJ by @fredericl", "retweets": 29.0 }, +{ "user": "TechCrunch", "text": "Groupon's Leadership Shuffle Continues: New North American Sales Head Named http://t.co/V4USZ7XU by @ingridlunden", "retweets": 19.0 }, +{ "user": "TechCrunch", "text": "Ptch Partners With Paramount, Letting Users Remix Their Own Paranormal Activity Found Footage http://t.co/28vTlzvu by @ryanlawler", "retweets": 17.0 }, +{ "user": "TechCrunch", "text": "Foursquare Partners With OpenTable To Make Reservations Simple http://t.co/EUz36s6i by @thatdrew", "retweets": 82.0 }, +{ "user": "TechCrunch", "text": "Treasure Data Launches Cloud-Based Data Warehouse With Investment From Ruby Creator Yukihiro ?Ma... http://t.co/Ds9mYFAX by @alexwilliams", "retweets": 15.0 }, +{ "user": "TechCrunch", "text": "Meet Apple's \"New CEO,\" Tom Brooks http://t.co/fdyX3Oox by @drizzled", "retweets": 54.0 }, +{ "user": "TechCrunch", "text": "Music-Sharing Startup MyStream Preps Android Launch, Looks Beyond Music http://t.co/CusbX5mh by @anthonyha", "retweets": 18.0 }, +{ "user": "TechCrunch", "text": "HP's Open webOS Goes 1.0, Can Now Be Ported To New Devices http://t.co/xljTC0L7 by @drizzled", "retweets": 60.0 }, +{ "user": "TechCrunch", "text": "Another Day, Another Eyeful Of HTC's Mildly-Updated One X+ http://t.co/LLU27dkp by @chrisvelazco", "retweets": 16.0 }, +{ "user": "TechCrunch", "text": "Color CEO Bill Nguyen Checks Out Of Day-To-Day Operations, While A New Leadership... http://t.co/vePSKt9x by @loyalelectron and @ryanlawler", "retweets": 19.0 }, +{ "user": "TechCrunch", "text": "Professor Facebook: Social Network Co-Designs New Course At General Assembly http://t.co/3UvQ3Mts by @ferenstein", "retweets": 19.0 }, +{ "user": "TechCrunch", "text": "Windows Phone Dev Center Now Provides User Review Translations For App Developers http://t.co/shAUIS64 by @drizzled", "retweets": 15.0 }, +{ "user": "TechCrunch", "text": "Sprint And Motorola Party Like It's 2010: Motorola XPRT Finally Gets A Taste Of Gingerbread http://t.co/vOJRwng8 by @romaindillet", "retweets": 7.0 }, +{ "user": "TechCrunch", "text": "U.S. Appeals Court: Samsung Free To Seek Lifting Of Galaxy Tab 10.1 Sales Injunction http://t.co/HTvIDlQU by @drizzled", "retweets": 24.0 }, +{ "user": "TechCrunch", "text": "Google Launches A New Tablet-Optimized User Interface For Flight Search http://t.co/xQOdAA60 by @fredericl", "retweets": 36.0 }, +{ "user": "TechCrunch", "text": "Twitter Announces New Improvements For Discover Tab, Adds Continuous Tweet Stream http://t.co/DpMRuDvO by @thatdrew", "retweets": 69.0 }, +{ "user": "TechCrunch", "text": "Apple Is Heavily Promoting Alternative Map Apps On The App Store http://t.co/LNf6lKQH by @thatdrew", "retweets": 77.0 }, +{ "user": "TechCrunch", "text": "PadPivot NST Review: The Best Available iPad Stand Just Got Better http://t.co/YJWkwkDd by @drizzled", "retweets": 11.0 }, +{ "user": "TechCrunch", "text": "AngelList Wings Is A Handy App For Searching AngelList On Your iPhone http://t.co/tJwqOMdi by @sarahintampa", "retweets": 7.0 }, +{ "user": "TechCrunch", "text": "Google Updates Gmail For iOS With Support For The iPhone 5's Larger Screen http://t.co/4fQ7YqYn by @fredericl", "retweets": 41.0 }, +{ "user": "TechCrunch", "text": "Well Played On Maps, Apple. Your Move, Google. http://t.co/kJvuAfw9 by @thatdrew", "retweets": 70.0 }, +{ "user": "TechCrunch", "text": "Flat-Fee, Dongle-free Card Payments Startup Emu Opens For Business In U.K., Registrations In Europe http://t.co/cILxZAVn by @riptari", "retweets": 27.0 }, +{ "user": "TechCrunch", "text": "Report: Apple's Pandora-Killer Hits Licensing Trouble With Sony/ATV http://t.co/7sV1BMb8 by @drizzled", "retweets": 55.0 }, +{ "user": "TechCrunch", "text": "LinkedIn Shuts Off API Access To Fellow Professional Social Network Viadeo For TOS Violations http://t.co/nAFHWLQ4 by @leenarao", "retweets": 31.0 }, +{ "user": "TechCrunch", "text": "iOS 6 Adoption At Just Over One Week: 60% For iPhone And 41% For iPad http://t.co/Q0HAgCz8 by @drizzled", "retweets": 125.0 }, +{ "user": "TechCrunch", "text": "Business Dashboards Get Instant Widgets, And Geckoboard Gets $1.5M From A Group Of Top Investors http://t.co/lDMDJUMQ by @ingridlunden", "retweets": 29.0 }, +{ "user": "TechCrunch", "text": "BlackBerry 10 Touch, Qwerty Devices Leak In Video; RIM Wants Lady Gaga To Help Sell BB10 http://t.co/7rFGECgA by @riptari", "retweets": 36.0 }, +{ "user": "TechCrunch", "text": "Neil Young Begins His Long Quest Towards True Audio Fidelity With Pono, A New Music Service And Device http://t.co/t1Ez2d26 by @johnbiggs", "retweets": 34.0 }, +{ "user": "TechCrunch", "text": "Socialize's New Ad Platform Turns Passbook Passes Into Mobile Ads http://t.co/I2aku6yx by @sarahintampa", "retweets": 21.0 }, +{ "user": "TechCrunch", "text": "Pay-By-Keyword Startup Seconds Refocuses As A Way To Make Anyone A Merchant http://t.co/id0jy9i4 by @drizzled", "retweets": 19.0 }, +{ "user": "TechCrunch", "text": "Google Maps, Now With More High-Res Satellite And 45? Aerial Imagery http://t.co/FRIe230w by @mjburnsy", "retweets": 83.0 }, +{ "user": "TechCrunch", "text": "Tim Cook Apologizes For Apple Maps, Points To Competitive Alternatives http://t.co/aOONqQLt by @jordanrcrook", "retweets": 265.0 }, +{ "user": "TechCrunch", "text": "Apple's iPhone 5 Availability Expands: What It Means To Regional Carriers http://t.co/eNU1Mzfq by @drizzled", "retweets": 31.0 }, +{ "user": "TechCrunch", "text": "Survey: Younger Shoppers Increasingly Using Mobiles To Buy And Compare http://t.co/TzI9dyeM by @riptari", "retweets": 87.0 }, +{ "user": "TechCrunch", "text": "MyVR Raises $1.4M From SV Angel, Chris Dixon To Attack The $85B Vacation Rental Industry http://t.co/h65xm9mV by @ripemp", "retweets": 29.0 }, +{ "user": "TechCrunch", "text": "New Pricing For Amazon RDS Running Oracle Database http://t.co/nwXG81fu by @alexwilliams", "retweets": 33.0 }, +{ "user": "TechCrunch", "text": "Which Founders Use FoundersCard? Craig Newmark, Kevin Rose, Leah Busque & 8K Others http://t.co/krGynNsV by @ripemp", "retweets": 21.0 }, +{ "user": "TechCrunch", "text": "If You Had Remote Access To Your Neighbor's Printer, What Would You Print? http://t.co/2iXhjjNY by @alexia", "retweets": 94.0 }, +{ "user": "TechCrunch", "text": "Pluralis Takes A Crowdsourced Approach To Improving Your Landing Pages http://t.co/YcqHJuC7 by @anthonyha", "retweets": 27.0 }, +{ "user": "TechCrunch", "text": "Best photobomb of the day. http://t.co/Xa2EYTyd", "retweets": 326.0 }, +{ "user": "TechCrunch", "text": "An Afternoon With Bad Piggies, Rovio's Next Hit http://t.co/KpYlmFSQ by @jordanrcrook", "retweets": 18.0 }, +{ "user": "TechCrunch", "text": "Most Docks Should Work With The Lightning Adapter And iPhone 5 http://t.co/oGlTupcK by @johnbiggs", "retweets": 13.0 } +]""" + val engadget = """[ +{ "user": "engadget", "text": "Sony reveals Japan prices for Windows 8 VAIO machines - http://t.co/FRCu2XVb", "retweets": 18.0 }, +{ "user": "engadget", "text": "FreedomPop's pay-as-you-go data service launches in beta, offering 500MB of free WiMAX per month - http://t.co/Ny48yXUl", "retweets": 10.0 }, +{ "user": "engadget", "text": "Lenovo intros ThinkCentre M78 with AMD A-Series APU and a starting price of $449 - http://t.co/OEDe1EwW", "retweets": 22.0 }, +{ "user": "engadget", "text": "HP announces the ElitePad 900, a business-friendly Windows 8 tablet arriving in January - http://t.co/RjSj2cms", "retweets": 33.0 }, +{ "user": "engadget", "text": "How would you change Nokia's Lumia 900? - http://t.co/kW4MqiCv", "retweets": 8.0 }, +{ "user": "engadget", "text": "Amazon Kindle Paperwhite review - http://t.co/A1ytikHz", "retweets": 34.0 }, +{ "user": "engadget", "text": "Refresh Roundup: week of September 24th, 2012 - http://t.co/7BZkgDcN", "retweets": 5.0 }, +{ "user": "engadget", "text": "Slackware 14.0 now available, freshens and simplifies a Linux vanguard - http://t.co/Lq2s5RL6", "retweets": 33.0 }, +{ "user": "engadget", "text": "Slingbox 350 and 500 show up unannounced in Best Buy, flaunt 1080p and built-in WiFi - http://t.co/BCBmTq8i", "retweets": 32.0 }, +{ "user": "engadget", "text": "Switched On: iOS 6 gets back from the app - http://t.co/8j4YL4Yn", "retweets": 16.0 }, +{ "user": "engadget", "text": "IRL: Dyson DC44, NUU ClickMate PowerPlus and the Galaxy S III - http://t.co/5Duf2aa5", "retweets": 8.0 }, +{ "user": "engadget", "text": "B&N makes the Nook Simple Touch with GlowLight's $20 price drop official - http://t.co/qDOwJHFT", "retweets": 9.0 }, +{ "user": "engadget", "text": "New process for nanotube semiconductors could be graphene's ticket to primetime (video) - http://t.co/6hiwBora", "retweets": 30.0 }, +{ "user": "engadget", "text": "Inhabitat's Week in Green: ECOLAR house, transparent solar panel and Star Wars terrariums - http://t.co/oeINUocC", "retweets": 8.0 }, +{ "user": "engadget", "text": "Sony shuts down PSP Comic Store after October 30th, leaves most of us in the lurch for now - http://t.co/sYpIQLtO", "retweets": 30.0 }, +{ "user": "engadget", "text": "Google retires more services, consolidates others in continued efficiency bid - http://t.co/UUdCFYPD", "retweets": 53.0 }, +{ "user": "engadget", "text": "LightSquared pitches new plans to FCC in attempt to end GPS interference hex - http://t.co/MubfRh7q", "retweets": 10.0 }, +{ "user": "engadget", "text": "Razer Blade review (late 2012) - http://t.co/QJpAz1nu", "retweets": 31.0 }, +{ "user": "engadget", "text": "Maingear announces Nomad 17 gaming laptop, comes with custom paint job - http://t.co/mj9dT843", "retweets": 9.0 }, +{ "user": "engadget", "text": "Ask Engadget: should companies include a cable with a new product? - http://t.co/NpXV1CdY", "retweets": 6.0 }, +{ "user": "engadget", "text": "RT @engadgethd: Looking for more pictures of the new Logitech Harmony Touch universal remote? We've got 'em http://t.co/GL7D8DCO", "retweets": 34.0 }, +{ "user": "engadget", "text": "Target, Walmart list price drop for B&N's Nook Simple Touch with GlowLight to $119 - http://t.co/zLtZGjrK", "retweets": 13.0 }, +{ "user": "engadget", "text": "Mobile Miscellany: week of September 24th, 2012 - http://t.co/e4LWzDgb", "retweets": 9.0 }, +{ "user": "engadget", "text": "Logitech Harmony Touch remote pops up unannounced at Best Buy - http://t.co/PHOgiyAj", "retweets": 16.0 }, +{ "user": "engadget", "text": "Alt-week 9.29.12: 3D pictures of the moon, 4D clocks and laser-controlled worms - http://t.co/IuEKU297", "retweets": 10.0 }, +{ "user": "engadget", "text": "Hard drive shipments recover from floods in Thailand, expected to reach record high - http://t.co/JnKct0aV", "retweets": 65.0 }, +{ "user": "engadget", "text": "Cubify lets you skin, 3D print your own personal Android - http://t.co/S6nimh5R", "retweets": 23.0 }, +{ "user": "engadget", "text": "Facebook Messenger 2.0 for iPhone brings new design to conversations, swipe left for friends list - http://t.co/fDQO7eJN", "retweets": 55.0 }, +{ "user": "engadget", "text": "US Cellular expands its Windows Phone catalog with the ZTE Render for $80 - http://t.co/uPZzQyy8", "retweets": 13.0 }, +{ "user": "engadget", "text": "Modbook Pro to launch with SSD storage, up for pre-order October 3rd - http://t.co/w87nj1W4", "retweets": 61.0 }, +{ "user": "engadget", "text": "BlackBerry 10 L-series tutorial videos surface online, give a literal peek at the future (video) - http://t.co/fLLDSRvA", "retweets": 47.0 }, +{ "user": "engadget", "text": "Logitech promises continued support for Squeezebox, says it won't force a switch to UE Smart Radio - http://t.co/xGFz7aLE", "retweets": 17.0 }, +{ "user": "engadget", "text": "Scape, Brian Eno's new ambient music creation app is now available on the iPad (video) - http://t.co/pRgrXoHA", "retweets": 33.0 }, +{ "user": "engadget", "text": "Touch Bionics releases new prosthetic fingers, flip the old ones the bird - http://t.co/xJNFmahn", "retweets": 48.0 }, +{ "user": "engadget", "text": "Sony starts delivering Ice Cream Sandwich update to Xperia go, Xperia U and Xperia sola - http://t.co/ao0Y6Iur", "retweets": 38.0 }, +{ "user": "engadget", "text": "Xi3 goes the crowdfunding route for future X3A, X7A modular PCs (video) - http://t.co/aPvAO10A", "retweets": 14.0 }, +{ "user": "engadget", "text": "FCC votes in favor of rethinking spectrum holding rules, goading broadcasters into wireless selloffs - http://t.co/x50qKCNB", "retweets": 14.0 }, +{ "user": "engadget", "text": "Prometheus Blu-ray specs unveiled, arrives with seven hours of extras October 9th (video) - http://t.co/xGnog3Kr", "retweets": 60.0 }, +{ "user": "engadget", "text": "Samsung Galaxy Note II variants for AT&T, T-Mobile, Verizon possibly caught at the FCC - http://t.co/YLvixd9D", "retweets": 21.0 }, +{ "user": "engadget", "text": "Nokia Parking helps you find and pay for a spot, we go hands-on (video) - http://t.co/NnAvzIlG", "retweets": 43.0 }, +{ "user": "engadget", "text": "PSA: $25 Google Play credit for Nexus 7 ends this weekend - http://t.co/3iyUo8iK", "retweets": 32.0 }, +{ "user": "engadget", "text": "US Appeals court rules Motorola can't enforce injunction against Microsoft in Germany... again - http://t.co/K2efo4cU", "retweets": 9.0 }, +{ "user": "engadget", "text": "This is the carbon fiber core of Ferrari's first hybrid - http://t.co/y2HP1rjX", "retweets": 43.0 }, +{ "user": "engadget", "text": "For Nokia, helping the competition find its way is good business - http://t.co/1sSiTbn7", "retweets": 22.0 }, +{ "user": "engadget", "text": "Google optimizes Flight Search for tablets, makes booking trips easier - http://t.co/HufiiXw2", "retweets": 29.0 }, +{ "user": "engadget", "text": "Microsoft details its own Windows 8 rollout, lessons learned from 'dogfooding' - http://t.co/bg5swwsm", "retweets": 33.0 }, +{ "user": "engadget", "text": "PBS draws link between digital music ethics and magic spells, somehow makes it look simple (video) - http://t.co/Fx7sPphG", "retweets": 7.0 }, +{ "user": "engadget", "text": "PSA: iPhone 5 available in 22 more countries, on Cricket and US regional carriers galore - http://t.co/lE4dLNQC", "retweets": 18.0 }, +{ "user": "engadget", "text": "\"It's now a surprise to hear of a company listening to its users, issuing an apology and vowing to make it better.\" (http://t.co/R5bu5Wp1)", "retweets": 49.0 }, +{ "user": "engadget", "text": "Wikipad CEO James Bower defends his gaming tablet's $500 pricing, why one device beats two - http://t.co/eUiFdD8g", "retweets": 11.0 }, +{ "user": "engadget", "text": "Parallella project: designed to bring mouth-watering power to a board similar in size to the Raspberry Pi for $99. http://t.co/lOz8cmX8", "retweets": 12.0 }, +{ "user": "engadget", "text": "Kodak dropping out of the consumer inkjet printer business in 2013 - http://t.co/bpE6f6Za", "retweets": 20.0 }, +{ "user": "engadget", "text": ".@EngadgetDistro's Issue 59 is ready for your eyes and fingers. We've got bicycles, new Sony products and much more. (http://t.co/Xk2F1et0)", "retweets": 2.0 }, +{ "user": "engadget", "text": "Court of Appeals for the Federal Circuit tells Judge Koh to revisit Galaxy Tab 10.1 injunction - http://t.co/iIOCcwDW", "retweets": 13.0 }, +{ "user": "engadget", "text": "Indian government tells carriers to end 3G roaming pacts, doesn't stop to explain why - http://t.co/Wwkk4qXY", "retweets": 12.0 }, +{ "user": "engadget", "text": "HP takes Open webOS 1.0 live, shows it supersized on a TouchSmart (video) - http://t.co/VWd1tmd7", "retweets": 17.0 }, +{ "user": "engadget", "text": "How to set up your Raspberry Pi to play Atari 2600 games - http://t.co/U7qMnHvb", "retweets": 29.0 }, +{ "user": "engadget", "text": "Amazon breaks down its Kindle Paperwhite light technology (video) - http://t.co/ThAUTZNt", "retweets": 18.0 }, +{ "user": "engadget", "text": "NPD: Android users chew an average 870MB of cellular data per month, youngest gobble the most - http://t.co/tUHgRYn8", "retweets": 25.0 }, +{ "user": "engadget", "text": "T-Mobile leases 7,200 mobile towers to Crown Castle in a 28-year, $2.4 billion deal http://t.co/jrlV0qrE", "retweets": 7.0 }, +{ "user": "engadget", "text": "Insert Coin: The Parallella project dreams of $99 supercomputers - http://t.co/nb0vKSKM", "retweets": 14.0 }, +{ "user": "engadget", "text": "Don't mind the zero-emissions Mercedes fuel-cell car behind the invisible curtain (eyes-on video) - http://t.co/3EYNK9d8", "retweets": 14.0 }, +{ "user": "engadget", "text": "Editorial: Apple apologies actually aren't that infrequent, and that's okay - http://t.co/oQn1MnEz", "retweets": 39.0 }, +{ "user": "engadget", "text": "AT&T 4G LTE adds Galaxy Note 2, Galay Tab 2 10.1, Galaxy Express and Galaxy Rugby Pro to lineup - http://t.co/uvBFFMQO", "retweets": 12.0 }, +{ "user": "engadget", "text": "Engadget Podcast 311 - 09.28.2012 - http://t.co/rirxdjgi", "retweets": 1.0 }, +{ "user": "engadget", "text": "GreenGT H2 eyes-on: the first fuel cell-powered racer to tackle Le Mans - http://t.co/zYBo1SnD", "retweets": 9.0 }, +{ "user": "engadget", "text": "Google offers up more high-res places in Maps / Earth, intros additional 45-degree imagery - http://t.co/MV8SSHxb", "retweets": 29.0 }, +{ "user": "engadget", "text": "Distro Issue 59: Boston's DBC City Bike is putting a new spin on an old design - http://t.co/1KOqTPVy", "retweets": 2.0 }, +{ "user": "engadget", "text": "Tim Cook apologizes for Maps mess - http://t.co/8QaU630q", "retweets": 210.0 }, +{ "user": "engadget", "text": "Smart Forstars concept EV brings drive-in theater to a wall near you (video) - http://t.co/u9Hxmlf3", "retweets": 20.0 }, +{ "user": "engadget", "text": "John Rogers returns with a silicon-silk circuit that dissolves inside your body - http://t.co/5qUQnKnP", "retweets": 9.0 }, +{ "user": "engadget", "text": "Neil Young's Pono music service wants to democratize high-quality audio (video) - http://t.co/gncQNRiS", "retweets": 20.0 }, +{ "user": "engadget", "text": "Build-a-bear's new store concept wants you to choose, love, stuff and fluff with high-tech (video) - http://t.co/3YwQ6tPS", "retweets": 11.0 }, +{ "user": "engadget", "text": "Sony Xperia Tablet S gets chunky update: better multi-tasking and IR, guest mode, new media apps - http://t.co/6H2fl92W", "retweets": 18.0 }, +{ "user": "engadget", "text": "Sony makes Olympus rescue pact official with $645 million investment - http://t.co/kTZdhyLK", "retweets": 17.0 }, +{ "user": "engadget", "text": "Ibis hotels to have robots paint art while they track your sleep: no, that's not creepy at all (video) - http://t.co/jzRbMwms", "retweets": 19.0 }, +{ "user": "engadget", "text": "FAVI's $50 Streaming Stick adds apps, streaming services to any HDTV with Android 4.1 Jelly Bean - http://t.co/dL9geyBU", "retweets": 20.0 }, +{ "user": "engadget", "text": "Google gives users an easy out, adds YouTube to Takeout data transfer tool - http://t.co/nPpUgW4Q", "retweets": 14.0 }, +{ "user": "engadget", "text": "NASA's Curiosity rover finds ancient streambed on Mars, evidence of 'vigorous' water flow - http://t.co/NEFjCaVj", "retweets": 67.0 }, +{ "user": "engadget", "text": "Google adds CardDAV support to contacts for easier syncing with iOS and other third-party devices - http://t.co/Sx5oXOvQ", "retweets": 34.0 }, +{ "user": "engadget", "text": "Sharp takes out $4.6 billion loan while it continues restructuring - http://t.co/O9t9mROq", "retweets": 15.0 }, +{ "user": "engadget", "text": "The Dark Knight Rises Blu-ray officially set for December 4th, limited edition Bat cowl revealed - http://t.co/7S0LD6Gx", "retweets": 210.0 }, +{ "user": "engadget", "text": "iPhone 5 vs. Lumia 920 Image Stabilization. Check out the test - http://t.co/TAMu9eYV", "retweets": 131.0 }, +{ "user": "engadget", "text": "ADATA's got an 8.9mm thick portable USB 3.0 drive, limbos under the competition by a millimeter - http://t.co/uNHhcDWN", "retweets": 10.0 }, +{ "user": "engadget", "text": "ZTE Engage cleared to land at Cricket on October 2nd for $250 - http://t.co/G72tUGEc", "retweets": 7.0 }, +{ "user": "engadget", "text": "NFL Network's Thursday Night Football Xtra app comes to Android - http://t.co/cPvJKgdR", "retweets": 12.0 }, +{ "user": "engadget", "text": "Exagon Motors Furtive-eGT electric supercar eyes-on - http://t.co/lkh6ANAW", "retweets": 9.0 }, +{ "user": "engadget", "text": "ZTE shows off the V98 Windows 8 slate, we go hands-on - http://t.co/FYrccrr8", "retweets": 13.0 }, +{ "user": "engadget", "text": "Kingston Digital SDXC cards arrive with lower price, larger capacities - http://t.co/yYfPHQOX", "retweets": 13.0 }, +{ "user": "engadget", "text": "http://t.co/WT4zylJj working on NFC-equipped business cards, kicks off limited beta program today - http://t.co/MtAea0jh", "retweets": 35.0 }, +{ "user": "engadget", "text": "Simple.TV clarifies shipping dates, tells backers units will start arriving October 10th - http://t.co/eod6khQJ", "retweets": 3.0 }, +{ "user": "engadget", "text": "Samsung Galaxy Camera swings past the FCC with AT&T-capable 3G - http://t.co/4lRRnTr3", "retweets": 9.0 }, +{ "user": "engadget", "text": "Mozilla Persona sign-in launches in beta, skips the social networking ball and chain (video) - http://t.co/GpVDM7Vz", "retweets": 8.0 }, +{ "user": "engadget", "text": "Cellcom to offer iPhone 5 for $149 and up starting Friday - http://t.co/4zCHVb0f", "retweets": 19.0 }, +{ "user": "engadget", "text": "Facebook Messenger for BlackBerry reaches App World, sates compulsive chatters - http://t.co/ZmuNQLjs", "retweets": 20.0 }, +{ "user": "engadget", "text": "RIM: first BlackBerry 10 devices to focus on mid-range and up, entry-level may be ready in 2013 - http://t.co/OMQMLJvs", "retweets": 34.0 }, +{ "user": "engadget", "text": "RT @EngadgetDistro: Since iOS 6's launch, how happy are you with Apple's Maps app?", "retweets": 25.0 }, +{ "user": "engadget", "text": "Harman shows off its upcomming JBL docks and speakers, we go eyes-on - http://t.co/5bz8UCBo", "retweets": 7.0 }, +{ "user": "engadget", "text": "From the lab: Lumia 920 image stabilization and 808 drop test at Nokia R&D (video) - http://t.co/s9zFp7hU", "retweets": 13.0 }, +{ "user": "engadget", "text": "Sony's Xperia T is now available on O2 and ThreeUK, Walther PPK/S not included - http://t.co/LlZsSzqQ", "retweets": 7.0 } +]""" + val amazondeals = """[ +{ "user": "amazondeals", "text": "Deal of the Day: 50% Off Select BEARPAW Boots & Slippers http://t.co/ZBvpSN33", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $38.69 - Honeywell Portable Wireless Door Chime and Push Button http://t.co/yYBV4ebr", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $124.99 - Ritmo Mundo Unisex White Sport Quartz Watch http://t.co/RfungjJC", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $129.99 - Giulio Romano Piemonte Black Silicone Watch http://t.co/7v1ovQCQ", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $59.99 - Stuhrling Original Men's Roulette Swiss Quartz Watch http://t.co/vreVCFvJ", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Marware Microshell for Fire (not for HD) http://t.co/Mtb1Uoea", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Belkin Quilted Case for Fire HD 7\" http://t.co/gxpUszgw", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $17.99 - iHome Rechargeable Speaker for Kindle Fire http://t.co/JCIBkwS1", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $119.95 - Shun Premier 7-Inch Santoku Knife http://t.co/Lpsyh0gY", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - Crock-Pot Cook Travel Serve 6-Quart Programmable Slow Cooker http://t.co/8HlHPMh3", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $37.49 - Tron: Legacy/Tron Original Classic (Five-Disc Combo: Blu-ray 3D / Blu-ray / DVD / Digital Co http://t.co/RqqRAwxO", "retweets": 8.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Dove Body Wash, 24 ounce (Pack of 4) http://t.co/PFmNhvsQ", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $249.99 - DwellStudio Crib Set, Owls http://t.co/f2C4ab6j", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $89.99 - Canon imageCLASS Laser Multifunction Printer http://t.co/RAiMddij", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Marware Jurni for Kindle + Kindle Paperwhite http://t.co/TzliFyFm", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $19.99 - Amazon Kindle DX Leather Cover http://t.co/BRDBNBsz", "retweets": 4.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Verso Duct Tape Case for Fire HD 7\" http://t.co/UID1EZfo", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Lightwedge Reading Light for Kindle E-readers http://t.co/8WvOPwib", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $1.99 Kindle Mysteries & Thrillers http://t.co/ApOcFD5g", "retweets": 7.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Lewis N. Clark Sport Pack http://t.co/ax9n0p4I", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Sherpani Latitude LE Wheeled Suitcase http://t.co/oor6wzfr", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $79.99 - Hunter 3.5-Gallon CareFree Plus Humidifier with PermaWick Filter http://t.co/TXnzAtGi", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $89.95 - Chicago Cutlery 12-Piece Knife Set with Block http://t.co/MFtohB5H", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $189.99 - 3M Mobile Projector http://t.co/pjwDE8s3", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $9.99 - Scotch Adhesive Dot Roller Value 4-Pack http://t.co/m7FdEzXr", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $89.99 - Swingline Stack-and-Shred Red 60-Sheet Shredder, Cross-Cut, 60 Sheets http://t.co/kIB7eczO", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $249.00 - Sony Cyber-shot 18.2 MP Digital Camera http://t.co/uy6V6XfB", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $59.99 - Jamie Oliver by T-fal Nonstick Hard Anodized 10-Piece Cookware Set http://t.co/C3Kqkhwi", "retweets": 4.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $199.95 - Cutlery Saber F-11 Chef Knives with Knife Bag http://t.co/EjdYR668", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Tommy Hilfiger Pebble Leather Easy Tote http://t.co/AAKERLrs", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.00 - AudioSource Indoor/Outdoor Two-Way Speakers http://t.co/4AMrXZYG", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $159.00 - AudioSource S3D60 Soundbar with Sonic Emotion 3D Sound http://t.co/2MpkXteo", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $99.99 - Jamo S-602 Bookshelf Speaker Pair http://t.co/qsK9vPee", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $41.35 - Keeper Waterproof Roof Top Cargo Bag http://t.co/gPJ6HEqs", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $81.35 - Masterbuilt Hitch-Haul Cargo Carrier http://t.co/CmBPGfnb", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $59.99 - T3 Mane Tamer Ionic Ceramic Tourmaline Flat Iron http://t.co/Nd7myReH", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $160.99 - Sarah Peyton 10-Inch Twin-Size Memory Foam Mattress http://t.co/3PHWkPcG", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $24.99 - PG Tips tea http://t.co/zwwzDoaV", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $17.99 - Mr. Beams Motion-Sensing Wireless LED Ceiling Light http://t.co/H67DlkoE", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $49.72 - Maytag Pur Refrigerator Cyst Water Filter 2-Pack http://t.co/Y3QuaVmb", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Jabra CLIPPER Bluetooth Stereo Headset http://t.co/XkvVrWJ0", "retweets": 5.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $219.99 - Dallas: The Complete Collection (Seasons 1-14 + Movies) http://t.co/bEIlKv37", "retweets": 5.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $89.99 - Ear Force DPX21 Headset http://t.co/lmrl7A47", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! LEGO Batman 2: DC Super Heroes http://t.co/Khqr9vao", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $19.99 - Plantronics Gamecom P90 http://t.co/Dd1SUbTy", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $34.99 - PS3 Black DualShock 3 Wireless Controller http://t.co/fG0zuhW4", "retweets": 6.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Sleeping Dogs http://t.co/4Hp1acZA", "retweets": 4.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Ghost Recon Future Soldier http://t.co/EP6t5M7F", "retweets": 10.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $9.99 - Black Eyed Peas Experience http://t.co/7uIfrnlp", "retweets": 5.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $39.99 - Madden NFL 13 http://t.co/YVUYk52D", "retweets": 18.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $14.99 - Death & Taxes (1 Page Book) http://t.co/QFepRypH", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $17.99 - Hello Kitty Sterling Silver Red Enamel Stud Earrings http://t.co/XG8ACVni", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Sterling Silver Created Gemstone and Diamond Jewelry Set http://t.co/swzagwJB", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $199.99 - Sterling Silver 1/2 cttw Black Diamond Bamboo Hoop Earrings http://t.co/ZF1r622t", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $24.99 - Sterling Silver Diamond Accent Dragonfly Pendant Necklace http://t.co/v8lzl73r", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Alex Stevens Men's Hoodie http://t.co/U00M1xj4", "retweets": 5.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Dickies Men's Short Sleeve Ring-Spun Work Shirt http://t.co/iTwRTTBL", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Get Ready for Fall: Save on Your Choice of Comforters http://t.co/lFFGKbuA", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - Baggallini Luggage Run Away Satchel Bag http://t.co/543etCwD", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $129.99 - Sennheiser OMX 980 In-Ear Headphones http://t.co/JgRjVQAU", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $17.99 - JLab TV Wall Mount http://t.co/ISYTXU5d", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $25.99 - Monster GreenPower Digital PowerCenter MDP 900 http://t.co/pKTK3bB7", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $12.99 - Atlantic Movie Bin http://t.co/0VMfGZUF", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $94.99 - Up to 66% off Schlage Camelot Keypad Lever Door Locks http://t.co/yBGB1JgW", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $23.55 - Christian Audigier Ed Hardy Deluxe Collection 4 Set Perfume http://t.co/hh6glIGv", "retweets": 4.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $59.99 - Panasonic DECT 6.0 Corded/Cordless Phone http://t.co/P1w5SlDz", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $99.99 - Polar SD Heart Rate Monitor Watch with S1 Foot Pod http://t.co/ckSeWPP8", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $84.99 - Cuisinart Petit Gourmet Portable Tabletop Gas Grill http://t.co/lRj6ZdPb", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $41.20 - Terrazzo Patio Table & Chair Set Cover http://t.co/J6qBBl9O", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $23.50 - Classic Accessories Veranda Patio Chair Cover http://t.co/zH75UlBK", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: Up to 54% off Select Exercise Bikes from Schwinn http://t.co/nx0MyVxf", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - Business Plan Pro Complete v 12 http://t.co/HpasTV4k", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $54.99 - Home Designer Suite 2012 by Chief Architect http://t.co/SWEoEFPL", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $24.99 - Uniden PRO505XL 40-Channel CB Radio http://t.co/K1skXu8V", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $19.99 - Garmin Carry All Case for Garmin nuvi Models http://t.co/qa8Sr9gL", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $154.99 - Garmin Forerunner 405CX GPS Watch with Heart Rate Monitor http://t.co/wV3vwncz", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $12.99 - Pyrex Storage 10-Piece Set http://t.co/NfJms2Dz", "retweets": 3.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $99.99 - 10-Piece KitchenAid Stainless Steel Cookware Set http://t.co/Dtl5zK5N", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $21.95 - Da Vinci Catapult Kit, Wood http://t.co/LUvEnJgo", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $117.02 - Norton Waterstone Starter Kit: 220/1000 grit stone, 4000/8000 grit stone, SiC flattening sto http://t.co/gMhOyN5z", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $294.99 - Panasonic 3D Blu-Ray Disc 5.1 Surround Sound Home Theater System http://t.co/vkAUPbOW", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Skechers Men's Rival Running Shoe http://t.co/leAWLm5F", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Bear Grylls Men's Mountain Jacket by Craghoppers http://t.co/TLQ1hg45", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - iHome Alarm Clock Dock for Kindle Fire (Not for HD) http://t.co/oDP4ZtSl", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $34.99 - SINGER Perfect Finish 1700 Watt Steam Iron http://t.co/wiri14KB", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - Armitron Men's Black/Rosegold Multi-Function Watch http://t.co/b1TXRG94", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $54.99 - Marc Ecko Men's The Encore OZ Classic Analog Watch http://t.co/ACe4nUE8", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $109.99 - Stuhrling Men's Raptor Mechanical Skeleton Watch http://t.co/S7YTLGYW", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - Stuhrling Original Men's Regatta Swiss Quartz Black Watch http://t.co/R050xFJK", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: $64.99 - Irwin Vise-Grip GrooveLock 8-Piece Plier Set http://t.co/noygxOJi", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $31.99 - Spynet Ultravision http://t.co/ThDnpqlC", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $19.99 - Lucky Brand Gold-Tone Turquoise-Color Howlite Link Bracelet http://t.co/w9jbDbZq", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $14.99 - Sterling Silver Plated Heart Cable Bangle Bracelet http://t.co/oRPIiEae", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $79.99 - Duragold 14k Yellow Gold Diamond-Cut Hoop Earrings http://t.co/yKlvrs2O", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! Sterling Silver Diamond Stud Earrings http://t.co/8dzWkf2r", "retweets": 1.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $9.99 - Brother PT90 Personal Labeler http://t.co/pJuiSxZn", "retweets": 0.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $39.99 - TRENDnet 200 Mbps Powerline Ethernet AV Adapter Kit http://t.co/AhwHHIUD", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $299.99 - Neato XV-11 All-Floor Robotic Vacuum System http://t.co/jQTTi3Im", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Lightning Deal! $101.15 - BISSELL Lift-Off Steam Mop http://t.co/kE5YhYZZ", "retweets": 2.0 }, +{ "user": "amazondeals", "text": "Deal of the Day: Starting from $3.99 Each: Essential MP3 Albums for Your Collection http://t.co/wotNjjir", "retweets": 5.0 } +]""" + val CNET = """[ +{ "user": "CNET", "text": "HP aims for business users with ElitePad 900 Windows 8 tablet http://t.co/XyOipAdL", "retweets": 10.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Apple rolls out fix for iPhone 5 Wi-Fi network data bug http://t.co/Gz13JfDD", "retweets": 93.0 }, +{ "user": "CNET", "text": "Ever wonder how your iPhone screen got its color? (Video) http://t.co/Fq9FFbnc", "retweets": 29.0 }, +{ "user": "CNET", "text": "The Kindle Paperwhite rises to the top of the e-reader pack http://t.co/QVXp6P0H", "retweets": 19.0 }, +{ "user": "CNET", "text": "How to switch from iPhone to Android http://t.co/M8I9lwua", "retweets": 131.0 }, +{ "user": "CNET", "text": "Let's crack open the iPhone 5! (video) http://t.co/4tiuroNg", "retweets": 30.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Oracle taps Nokia for location-based services, The Wall Street journal reports. http://t.co/ecxuXHD3", "retweets": 54.0 }, +{ "user": "CNET", "text": "How to simplify your messy photo collection http://t.co/nFwJ8jSi", "retweets": 24.0 }, +{ "user": "CNET", "text": "Everything you need to know about using iOS 6 http://t.co/10jWoN7I #ICYMI #Ios6", "retweets": 47.0 }, +{ "user": "CNET", "text": "Where iPhones go to die (video) http://t.co/86fBpfvh", "retweets": 33.0 }, +{ "user": "CNET", "text": "How RIM could get you to buy into BlackBerry 10 http://t.co/AUo08CkG", "retweets": 28.0 }, +{ "user": "CNET", "text": "Camera contest: Apple iPhone 5 vs. Samsung Galaxy S3 vs. HTC One X http://t.co/PmbhNgrd", "retweets": 49.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Apple backtracks on 'most powerful' map app claim http://t.co/mB39ilOg", "retweets": 50.0 }, +{ "user": "CNET", "text": "Digging for rare earths: The mines where iPhones are born http://t.co/TKHx8hYK", "retweets": 29.0 }, +{ "user": "CNET", "text": "The new Twitter for iOS adds headline photos and revamps iPad interface. Check out our review: http://t.co/234thJkl", "retweets": 17.0 }, +{ "user": "CNET", "text": "Yes, a search engine just for porn http://t.co/AhFCUf8I", "retweets": 60.0 }, +{ "user": "CNET", "text": "How to reduce annoying microphonics from your in-ear headphones http://t.co/dmwHA4LY", "retweets": 22.0 }, +{ "user": "CNET", "text": "Is this what an iPad Mini might look like? http://t.co/MmfrLItm", "retweets": 50.0 }, +{ "user": "CNET", "text": "The mad world of Foxconn, your iPhone's birthplace: @iamjaygreene reports from China http://t.co/WsUssqj7 #ICYMI", "retweets": 45.0 }, +{ "user": "CNET", "text": "Sony's new portable external battery is gorgeous http://t.co/upKqXII7", "retweets": 42.0 }, +{ "user": "CNET", "text": "FIFA Soccer 13 is not only the best soccer game, but the best sports game on an iOS device. Check out our review: http://t.co/pZv7DKF1", "retweets": 63.0 }, +{ "user": "CNET", "text": "The wireless router is arguably the most important piece of computer equipment in your home. You deserve a good one http://t.co/LQCO1VBz", "retweets": 88.0 }, +{ "user": "CNET", "text": "The periodic table of iPhones (infographic) http://t.co/ShUvE27y", "retweets": 54.0 }, +{ "user": "CNET", "text": "The environmental pitfalls at the end of an iPhone's life http://t.co/jLfiPCVA", "retweets": 43.0 }, +{ "user": "CNET", "text": "Transit directions field test: Apple Maps vs. Google Maps http://t.co/M9kDqLGX", "retweets": 36.0 }, +{ "user": "CNET", "text": "Will the Nook HD+ put out the Kindle Fire? Check out our hands on with Barnes & Noble's latest tablet http://t.co/t6A2EV1D", "retweets": 10.0 }, +{ "user": "CNET", "text": "How to wake up to any song in your iTunes library in iOS 6 http://t.co/hTnUiyjt", "retweets": 68.0 }, +{ "user": "CNET", "text": "How to lock down and find Android and Windows phones http://t.co/mRw8P80z", "retweets": 25.0 }, +{ "user": "CNET", "text": "Four most-useful new settings in iOS 6 http://t.co/LHFOCLnA", "retweets": 63.0 }, +{ "user": "CNET", "text": "Looper review: Bruce Willis does the time warp, kicks ass http://t.co/ttIk26r8", "retweets": 19.0 }, +{ "user": "CNET", "text": "In need of a new app? Here are our favourites from this week http://t.co/9D3kEhWf", "retweets": 17.0 }, +{ "user": "CNET", "text": "How rocks power your iPhone (video) http://t.co/hufxUMok", "retweets": 11.0 }, +{ "user": "CNET", "text": "View Einstein's brain under a microscope, piece by piece http://t.co/b7O8YDDN", "retweets": 31.0 }, +{ "user": "CNET", "text": "If you're really, really, really paranoid about Facebook privacy issues, then maybe just leave Facebook http://t.co/fDPLjXHM", "retweets": 72.0 }, +{ "user": "CNET", "text": "How to prevent phone and tablet theft http://t.co/LbhmJT7t", "retweets": 62.0 }, +{ "user": "CNET", "text": "Whether you're a casual shopper or an avid buyer & seller, eBay for Android is worth a download. Check out our review http://t.co/dNWIjcxx", "retweets": 10.0 }, +{ "user": "CNET", "text": "Twitter's revamped Android app is wonderful, but is it enough to satisfy power users? Check out our full review: http://t.co/234thJkl", "retweets": 25.0 }, +{ "user": "CNET", "text": "Google cofounder Sergey Brin: Robotic cars will be available to the general public within 5 years http://t.co/al2PyAP5", "retweets": 134.0 }, +{ "user": "CNET", "text": "The original Kindle Fire goes through the battery of Torture Tests. Will it survive? http://t.co/O0gEUklc", "retweets": 10.0 }, +{ "user": "CNET", "text": "iPhone 5 to iPhone 4S: I vibrate so much harder than you http://t.co/1XMZVvVE", "retweets": 57.0 }, +{ "user": "CNET", "text": "Take a look at how Lenovo stress-tests laptops http://t.co/MQCMeVBz", "retweets": 32.0 }, +{ "user": "CNET", "text": "FIFA Soccer 13 is not only the best soccer game, but the best sports game on an iOS device. Check out our review: http://t.co/pZv7DKF1", "retweets": 59.0 }, +{ "user": "CNET", "text": "The Sonos Connect brings effortless wireless media streaming to existing hi-fi owners. Check out our review: http://t.co/xyHxFLXi", "retweets": 9.0 }, +{ "user": "CNET", "text": "The cheap, smart Vizio 32-inch TV is a great buy. Check out our review: http://t.co/AjH2yFgQ", "retweets": 11.0 }, +{ "user": "CNET", "text": "The Super Slim PlayStation 3 shrinks a powerful gaming machine into an even tinier package. Check out our review: http://t.co/1LUyyuEj", "retweets": 35.0 }, +{ "user": "CNET", "text": "RT @benjaminphotos: @CNETNews Yes we are... http://t.co/NxGMjVWq", "retweets": 74.0 }, +{ "user": "CNET", "text": "Inside Scoop: @Josh discusses the ins and outs of Apple's recent map flub (video) http://t.co/4wLrkvFG #mapology", "retweets": 2.0 }, +{ "user": "CNET", "text": "Is the computer mouse dead? @DanAckerman on the ascendance of the touch pad: http://t.co/vjwps44j", "retweets": 43.0 }, +{ "user": "CNET", "text": "If you watch television regularly, second-screen app Zeebox for Android & iOS makes an excellent companion http://t.co/buVYA8E7", "retweets": 15.0 }, +{ "user": "CNET", "text": "MacFixIt Q&A: Is your iTunes not reading burned disks? http://t.co/GnPo0lLv", "retweets": 6.0 }, +{ "user": "CNET", "text": "Business cards go wireless as Moo folds in NFC technology. Bump goes the contact info http://t.co/N6zGZUdF", "retweets": 27.0 }, +{ "user": "CNET", "text": "The awesome size of Foxconn (infographic) http://t.co/drvvmsVb", "retweets": 19.0 }, +{ "user": "CNET", "text": "The day the Earth stood still... while Tim Cook apologized to it. Get the full recap on @CNETUpdate http://t.co/rqJjKXph", "retweets": 23.0 }, +{ "user": "CNET", "text": "How to download your YouTube videos in their original format (video) http://t.co/J7QOQiul", "retweets": 42.0 }, +{ "user": "CNET", "text": "Are you ready for Facebook updates to invade your caller ID? http://t.co/uG8lG9J6", "retweets": 28.0 }, +{ "user": "CNET", "text": "The cost of charging your iPhone 5 for one year: $0.41 http://t.co/DTOV8D3M", "retweets": 93.0 }, +{ "user": "CNET", "text": "The iPhone map of China (infographic) http://t.co/ioazYk6y", "retweets": 13.0 }, +{ "user": "CNET", "text": "Finally, you can play as the pigs from Angry Birds! But is 'Bad Piggies' as addictive as the original game? (review) http://t.co/7W6vflLf", "retweets": 15.0 }, +{ "user": "CNET", "text": "When an iPhone is recycled (infographic) http://t.co/YixSaMOi", "retweets": 18.0 }, +{ "user": "CNET", "text": "A mathematician accurately predicted when Android's app store would hit 25 billion downloads http://t.co/VFLBJ0z3", "retweets": 36.0 }, +{ "user": "CNET", "text": "Tech tip: Manage your iPhone address book with CopyTrans Contacts http://t.co/C3Je5FJo", "retweets": 14.0 }, +{ "user": "CNET", "text": "Man jailed for mistakenly sexting young girls, freed because he also inadvertently sexted his own family http://t.co/Mp11ELmH", "retweets": 33.0 }, +{ "user": "CNET", "text": "A TSA security guard was caught red-handed stealing an iPad in a TV sting operation http://t.co/HipBLbgs", "retweets": 43.0 }, +{ "user": "CNET", "text": "The periodic table of iPhones (infographic) http://t.co/ShUvE27y", "retweets": 21.0 }, +{ "user": "CNET", "text": "How to opt out of Facebook's online and offline tracking http://t.co/VCJnZdM7", "retweets": 70.0 }, +{ "user": "CNET", "text": "How to make your Android phone look like an iPhone 5 http://t.co/tZZYb8Ti", "retweets": 39.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Judge Lucy Koh now has the authority to lift the sales ban on Samsung's Galaxy Tab 10.1 tablet. Will she? http://t.co/2SwBiOAE", "retweets": 19.0 }, +{ "user": "CNET", "text": "Getting started with Passbook on iOS 6 http://t.co/3ATZW25o", "retweets": 33.0 }, +{ "user": "CNET", "text": "Researchers are building a real life RoboCop! No word yet on if they'll follow it up with two inferior sequels http://t.co/a3lunSCQ", "retweets": 23.0 }, +{ "user": "CNET", "text": "Scenes from the 2012 Paris Motor Show http://t.co/z1MhqngC", "retweets": 15.0 }, +{ "user": "CNET", "text": "Minecraft creator says he won't certify the game for Windows 8 http://t.co/xqo84j2B", "retweets": 25.0 }, +{ "user": "CNET", "text": "How to change your Twitter header photo http://t.co/FCs8hpTs", "retweets": 27.0 }, +{ "user": "CNET", "text": "The tweets in the Discovery tab just got a lot more relevant http://t.co/v6IWlTdQ", "retweets": 5.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Tim Cook's \"Mapology\": What's missing is any explanation why Apple would issue a beta app it knew would frustrate you http ...", "retweets": 32.0 }, +{ "user": "CNET", "text": "How to set up an Android tablet as a second display for your PC or Mac http://t.co/YynJll9N", "retweets": 48.0 }, +{ "user": "CNET", "text": "A Texas company is joining the patent wars by suing Apple over spreadsheet technology http://t.co/sntXqimH", "retweets": 16.0 }, +{ "user": "CNET", "text": "Eying the iPad's turf, Intel and the Windows 8 gang is set to make a play for business users http://t.co/7OEr4hqc", "retweets": 7.0 }, +{ "user": "CNET", "text": "Friday Poll: Would you use a self-driving car? http://t.co/qRAvKCpc", "retweets": 7.0 }, +{ "user": "CNET", "text": "The mad world of Foxconn, your iPhone's birthplace: @iamjaygreene reports from China http://t.co/WsUssqj7 #ICYMI", "retweets": 19.0 }, +{ "user": "CNET", "text": "RT @CNETNews: As Apple says sorry, Google Maps gets a little better, depending on where you are http://t.co/6k2WPgc3", "retweets": 42.0 }, +{ "user": "CNET", "text": "Sure, tablets and smartphones are risky, but HP doesn't really have any choice but to plunge into the post-PC era http://t.co/vnTBjSMo", "retweets": 6.0 }, +{ "user": "CNET", "text": "RT @jetscott: The good news: New York's getting the world's tallest Ferris wheel. The bad news: it's on Staten Island. http://t.co/5VciVVue", "retweets": 19.0 }, +{ "user": "CNET", "text": "11 essential tips for mastering iOS 6 http://t.co/10jWoN7I", "retweets": 37.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Here's what AT&T's got coming up, a la Samsung: Galaxy Note 2, Rugby Pro, Express, Tab 10.1 2 http://t.co/iOwZxoRR", "retweets": 12.0 }, +{ "user": "CNET", "text": "Observers of a humans vs. bots death match decided that the bots were more human than the actual humans http://t.co/BXNwyH8a", "retweets": 11.0 }, +{ "user": "CNET", "text": "RT @jeskillings: Apple's iOS 6 maps apology today: gracious http://t.co/F25NOxr6 Apple's iPhone 4 antenna apology in 2010: grudging http ...", "retweets": 37.0 }, +{ "user": "CNET", "text": "Getting started with the YouTube app in iOS 6 http://t.co/HhACyxfp", "retweets": 12.0 }, +{ "user": "CNET", "text": "Distressed camera maker Olympus is getting a $645 million boost care of Sony http://t.co/VzKufK5z", "retweets": 8.0 }, +{ "user": "CNET", "text": "Deal alert: Get a free year of SiriusXM Internet radio! http://t.co/LXUrqZ1R", "retweets": 42.0 }, +{ "user": "CNET", "text": "Kindle Fire 2012: Even hotter than last year. Check out our full review of Amazon's updated media tablet http://t.co/lUxnqomZ", "retweets": 15.0 }, +{ "user": "CNET", "text": "Music publisher Sony/ATV killed Apple's iPhone 5 music service, report says http://t.co/whx2JbAO", "retweets": 22.0 }, +{ "user": "CNET", "text": "5 great, cheap plasma TVs http://t.co/4QYARxB8", "retweets": 10.0 }, +{ "user": "CNET", "text": "How RIM could get you to buy into BlackBerry 10 http://t.co/AUo08CkG", "retweets": 15.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Apple \"fell short\" with iOS 6 maps, and we are \"extremely sorry,\" CEO Tim Cook says in open letter http://t.co/t1U4497r", "retweets": 139.0 }, +{ "user": "CNET", "text": "iPhone 5 to iPhone 4S: I vibrate so much harder than you http://t.co/1XMZVvVE", "retweets": 43.0 }, +{ "user": "CNET", "text": "Leak: Photos of RIM's BlackBerry 10 phones hit the web http://t.co/X0OvqMwL", "retweets": 46.0 }, +{ "user": "CNET", "text": "RT @CNETNews: Apple's Tim Cook: We are \"extremely sorry\" about those problems with iOS 6 Maps http://t.co/rlcCUgQO", "retweets": 114.0 }, +{ "user": "CNET", "text": "Samsung Galaxy Music leak shows budget music phone http://t.co/aTAjDKk2", "retweets": 14.0 }, +{ "user": "CNET", "text": "Bad Piggies review: Angry Birds follow-up bolts on complexity http://t.co/lmfK0Yjd", "retweets": 16.0 }, +{ "user": "CNET", "text": "HTC One X+ turns up in O2 brochure, Jelly Bean in tow http://t.co/MGvjw24E", "retweets": 26.0 } +]""" + val gadgetlab = """[ +{ "user": "gadgetlab", "text": "A Clean, Well-Lighted Face: The Kindle Paperwhite http://t.co/WlUPzZbS by @strngwys", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "Maps? Purple halos in photos? Screen glitches? What? iPhone 5 'problems' explained http://t.co/xVGBVhGE by @redgirlsays", "retweets": 22.0 }, +{ "user": "gadgetlab", "text": "Tim Cook apologizes for Mapocalypse debacle, offers interim fixes http://t.co/ahPbLcU5 by @redgirlsays", "retweets": 18.0 }, +{ "user": "gadgetlab", "text": "Pretty Cities: Google Maps Improves Aerial Images http://t.co/nEopgNpY by @alexandra_chang", "retweets": 7.0 }, +{ "user": "gadgetlab", "text": "Nokia Lumia Premium Pricing Won?t Help Windows Phone Adoption: http://t.co/mbc7fu1a by @alexandra_chang", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "BlueStacks and AMD Bring 500,000 Android Apps to Windows 8: http://t.co/GskuXhRo by @alexandra_chang", "retweets": 22.0 }, +{ "user": "gadgetlab", "text": "BlackBerry 10 Could Be Too Little, Too Late http://t.co/ViQWBSYY By @Strngwys", "retweets": 28.0 }, +{ "user": "gadgetlab", "text": "And, in case you haven't seen it yet, @GadgetLab's review of iOS 6: http://t.co/PUf9Yisb by @redgirlsays", "retweets": 10.0 }, +{ "user": "gadgetlab", "text": "Our review of the iPhone 5: http://t.co/gkrfr1Bt by @redgirlsays", "retweets": 27.0 }, +{ "user": "gadgetlab", "text": "Ballmer Pitches Window 8 to Developers, But Keeps Tight Reign On Surface: http://t.co/TdkBZbXR by @alexandra_chang", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "Gadget Lab Show: Apple?s iPhone 5 Wins, Maps App Fails http://t.co/9gLGvagS with @redgirlsays + @strngwys", "retweets": 13.0 }, +{ "user": "gadgetlab", "text": "BlackBerry 10 features all-in-one approach to the inbox http://t.co/UzTPt3Dh by @strngwys", "retweets": 14.0 }, +{ "user": "gadgetlab", "text": "Update on @redgirlsays' #walletless adventures -- \"Living Walletless, Week One: My Kingdom for a Bagel\" http://t.co/3zEIPqdl", "retweets": 8.0 }, +{ "user": "gadgetlab", "text": "Foxconn rioting leaves factory closed, dozens injured http://t.co/VSgFcak5 by @redgirlsays", "retweets": 29.0 }, +{ "user": "gadgetlab", "text": "Who's Waiting in Line for the #iPhone5? These People Are: http://t.co/qeYJaGu2 by @redgirlsays", "retweets": 21.0 }, +{ "user": "gadgetlab", "text": "Operation iPhone Drop: From Cargo Plane to Door Stoop http://t.co/TEuxL13p By @strngwys", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "Know anyone around age 60 (parents, grandparents, yourself) looking to buy their first smartphone? Give @redgirlsays a ping!", "retweets": 4.0 }, +{ "user": "gadgetlab", "text": "iPhone 5 Exposed: iFixit Tears Down Apple's Latest http://t.co/4HbErQRw by @redgirlsays", "retweets": 35.0 }, +{ "user": "gadgetlab", "text": "Facebook?s New Plug-in Gives You Better Protection From Embarrassing Overshares: http://t.co/AbiHJWwJ by @alexandra_chang", "retweets": 10.0 }, +{ "user": "gadgetlab", "text": "Apple Mapocalypse Sends iOS 6 Users Into a Tizzy, Riverbank http://t.co/amGwTFUu by @redgirlsays", "retweets": 19.0 }, +{ "user": "gadgetlab", "text": "Don?t Miss the Bus, Gus: 7 Public Transit Apps (And One Workaround) for iOS 6 http://t.co/1mgyGbFC by @alexandra_chang", "retweets": 6.0 }, +{ "user": "gadgetlab", "text": "In case you missed it... Samsung continues Apple potshots with latest ad: http://t.co/IiNzQW7p by @redgirlsays", "retweets": 24.0 }, +{ "user": "gadgetlab", "text": "Multiple Wi-Fi Issues Plague iOS 6 Upgrade http://t.co/fI3LGTeY by @strngwys", "retweets": 42.0 }, +{ "user": "gadgetlab", "text": "Any Blackberry users (in the SF area) that are planning to switch to another smartphone platform? @alexandra_chang wants to talk.", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "Downloaded iOS 6 yet? Here are 13 tips, tricks and hidden features: http://t.co/IXWOHkXB by @strngwys and @redgirlsays", "retweets": 95.0 }, +{ "user": "gadgetlab", "text": "The details and specs on HTC's Windows Phone 8X and 8S: http://t.co/k8b0kQCZ by @alexandra_chang", "retweets": 14.0 }, +{ "user": "gadgetlab", "text": "HTC's 8X and 8S are \"Windows Phone's Best Shot\": http://t.co/2egTC11C by @alexandra_chang", "retweets": 10.0 }, +{ "user": "gadgetlab", "text": "Motorola?s Razr i: A Razr M With Intel Inside http://t.co/eLtTIxan by @nateog", "retweets": 19.0 }, +{ "user": "gadgetlab", "text": "Hi-Call Phone Gloves Bring Inspector Gadget Functionality to Life http://t.co/IbsRzeew by @nateog", "retweets": 33.0 }, +{ "user": "gadgetlab", "text": "Amazon Kindle Fire HD Teardown http://t.co/IR9rx6h3 by @nateog", "retweets": 35.0 }, +{ "user": "gadgetlab", "text": "Apple?s Jony Ive Designing One-Off Leica Camera http://t.co/s3PeVcUM by @karissabe", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "iFixit EarPod Teardown: Better Buds, But Don't Try Repairing Them http://t.co/IDtX4MBz by @redgirlsays", "retweets": 12.0 }, +{ "user": "gadgetlab", "text": "Galaxy Tab 10.1 Injunction Still Stands in Apple v. Samsung http://t.co/JwOCDnw1 by @redgirlsays", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "Hands-on with the redesigned Twitter for iPad http://t.co/JKLIVV4g by @redgirlsays", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "First iPhone 5 Benchmarks: Screaming Fast, Yes, But Just Shy of Galaxy S III http://t.co/QIAhda3L by @redgirlsays", "retweets": 79.0 }, +{ "user": "gadgetlab", "text": "Fitbit adds two new Bluetooth-compatible fitness trackers: The Fitbit Zip and Fitbit One http://t.co/38m1fUSR by @redgirlsays", "retweets": 4.0 }, +{ "user": "gadgetlab", "text": "This week on the Gadget Lab Show, iPhone 5 chat and hands on with Apple's EarPods: http://t.co/0DPubUFz", "retweets": 13.0 }, +{ "user": "gadgetlab", "text": "Microsoft News: Windows 8 Event and Hints on Surface Pricing: http://t.co/kswqw4gj by @alexandra_chang", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "Yowza: iPhone 5 tops two million in pre-orders in first 24 hours on sale. http://t.co/Nx2d9CBN by @redgirlsays", "retweets": 47.0 }, +{ "user": "gadgetlab", "text": "From earlier today: iPhone 5 pre-orders come flooding in despite numerous website hiccups http://t.co/tbyirA59 by @redgirlsays", "retweets": 18.0 }, +{ "user": "gadgetlab", "text": "The iPhone 5 Lightning adapter could be bad news for high-end docks. @strngwys reports: http://t.co/90eAf68y", "retweets": 16.0 }, +{ "user": "gadgetlab", "text": "Apple iPhone 5 Specs vs. the Competition?s: Which Will You Buy?: http://t.co/r6xALNpe by @alexandra_chang", "retweets": 33.0 }, +{ "user": "gadgetlab", "text": "Think the new iPhone is great, but kinda boring? Here are 5 things that would make us fall in love all over again: http://t.co/sKm9P78B", "retweets": 32.0 }, +{ "user": "gadgetlab", "text": "So long, Ping: Apple's shuttering its failed social network Sept. 30 http://t.co/wO75kCfd by @redgirlsays", "retweets": 21.0 }, +{ "user": "gadgetlab", "text": "Why Apple Made Three iPhone 5 Models and What That Means For You: http://t.co/ll39koPx by @alexandra_chang", "retweets": 25.0 }, +{ "user": "gadgetlab", "text": "Pricing and Availability: Why It's a Tough Spec for Everyone But Apple http://t.co/bYd27fYD by @redgirlsays", "retweets": 6.0 }, +{ "user": "gadgetlab", "text": "Apple Confirms iPhone 5 Pre-Orders Start at 12:01AM September 14 http://t.co/Cwenf7Zu By @strngwys", "retweets": 45.0 }, +{ "user": "gadgetlab", "text": "Hands on with the faster, lighter, longer iPhone 5: http://t.co/D5z6yu45 by @redgirlsays", "retweets": 31.0 }, +{ "user": "gadgetlab", "text": "RT @strngwys: Why the iPod classic will never go away. http://t.co/zhCQRfu3", "retweets": 8.0 }, +{ "user": "gadgetlab", "text": "What You Need to Know About Apple?s New iPhone 5 http://t.co/ArTQrO6b by @alexandra_chang", "retweets": 41.0 }, +{ "user": "gadgetlab", "text": "RT @redgirlsays: So, what do you think was the most exciting part of today's event? iPhone 5? iPod touch? nano? iTunes? Foo Fighters?", "retweets": 9.0 }, +{ "user": "gadgetlab", "text": "Apple's #iPhone5 event is now over. New iPods, new iTunes, new EarPods. All sorts of newness. See our LIVEBLOG here... http://t.co/5AtpHCfY", "retweets": 32.0 }, +{ "user": "gadgetlab", "text": "http://t.co/z7Dsi93S has been updated with all the new product details... http://t.co/8sH8jdas #iPhone5", "retweets": 42.0 }, +{ "user": "gadgetlab", "text": "RT @wired: Apple in-ear headphones now EarPods - Look like little aliens speaking in your ears. On sale today. http://t.co/PewWjvwX", "retweets": 88.0 }, +{ "user": "gadgetlab", "text": "RT @redgirlsays: Yes, that's the foo fighters. http://t.co/qgUpreGb", "retweets": 29.0 }, +{ "user": "gadgetlab", "text": "RT @redgirlsays: Wait, we're getting a Foo Fighters concert? I guess that's almost as good as Hot Chip that I missed last night?", "retweets": 6.0 }, +{ "user": "gadgetlab", "text": "Say goodbye to the iconic Apple Earbuds. A new headphone set is introduced...the Earpods http://t.co/op5vlGq4 #Apple #iPhone5 LIVEBLOG", "retweets": 31.0 }, +{ "user": "gadgetlab", "text": "New iPod Touch will be $299 for 32 GB & $399 for 64 GB, shipping sometime October http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 52.0 }, +{ "user": "gadgetlab", "text": "New iPod Touch is available in 5 colors: white, black, cyan, yellow & red http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 21.0 }, +{ "user": "gadgetlab", "text": "Siri will be on the new iPod Touch. Cool! http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 16.0 }, +{ "user": "gadgetlab", "text": "New iPod Touch has same 4-inch Retina display as #iPhone5 + A5 CPU making it the 1st dual-core Touch http://t.co/DwtKQkSu #Apple LIVEBLOG", "retweets": 25.0 }, +{ "user": "gadgetlab", "text": "New iPod Touch announced. It's just 6.1-mm thin. http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "New iPod Nano has 2.5-in multitouch display. Available in white, black, pink, green, blue, yellow & red http://t.co/DwtKQkSu #Apple #iPhone5", "retweets": 26.0 }, +{ "user": "gadgetlab", "text": "7th-gen iPod nano is rectangular, 5 mm thin, about 40% than iPod nano it replaces http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 12.0 }, +{ "user": "gadgetlab", "text": "New iTunes for desktops will launch late October http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 10.0 }, +{ "user": "gadgetlab", "text": "#iOS6 will feature new iTunes stores and a new desktop iTunes (w iCloud built in) will launch too http://t.co/DwtKQkSu #Apple #iPhone5", "retweets": 12.0 }, +{ "user": "gadgetlab", "text": "#iOS6 hits the iPhone 4S, 4, 3GS, 3rd-gen iPad, iPad 2 & 4th-gen iPod Touch on Sept. 19 http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 42.0 }, +{ "user": "gadgetlab", "text": "iPhone 4 (8 GB) is now free & 4S (16GB) is now $99 on 2-year carrier contracts http://t.co/DwtKQkSu #Apple #iPhone5 LIVEBLOG", "retweets": 32.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 pre-orders start Sept. 14. Hits stores Sept. 21. http://t.co/DwtKQkSu #Apple LIVEBLOG", "retweets": 79.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 costs the same as the iPhone 4S: $199 for 16 Gb, $200 for 32 GB, $399 for 64 GB http://t.co/DwtKQkSu #Apple LIVEBLOG", "retweets": 66.0 }, +{ "user": "gadgetlab", "text": "?iPhone 5 is the best phone we?ve ever made,? @PSchiller says http://t.co/DwtKQkSu #Apple LIVEBLOG", "retweets": 27.0 }, +{ "user": "gadgetlab", "text": "#Apple says #iPhone5 has a 4-inch screen you can still comfortably use with one hand http://t.co/DwtKQkSu LIVEBLOG", "retweets": 23.0 }, +{ "user": "gadgetlab", "text": "#Apple #iPhone5 comes in either Slate (aka black) or White http://t.co/DwtKQkSu LIVEBLOG", "retweets": 17.0 }, +{ "user": "gadgetlab", "text": "#Apple: Shared Photo Streams in #iOS6 is \"easiest way to share photos with your friends? http://t.co/DwtKQkSu #iPhone5 LIVEBLOG #iPhone5", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "#Apple now talking iOS 6 updates for the #iPhone5 http://t.co/DwtKQkSu LIVEBLOG", "retweets": 25.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 Lightning connector is 80% smaller than old 30-pin connectors http://t.co/DwtKQkSu LIVEBLOG #Apple", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 has smaller, new connecter called \"Lightning\" in a nod to Thunderbolt ports on Macs http://t.co/DwtKQkSu LIVEBLOG #Apple", "retweets": 32.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 front camera shoots 720p, has Face detection & #Apple announces FaceTime over cellular networks http://t.co/DwtKQkSu LIVEBLOG", "retweets": 94.0 }, +{ "user": "gadgetlab", "text": "#Apple #iPhone5 has 40% faster photo capture thanks iPhone 4S http://t.co/DwtKQkSu LIVEBLOG", "retweets": 17.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 has camera has new image sensor for better low light photos & is 25% smaller http://t.co/DwtKQkSu LIVEBLOG #Apple", "retweets": 21.0 }, +{ "user": "gadgetlab", "text": "#Apple #iPhone5 camera is 8megapixels...same as iPhone 4S. But it's not the same camera http://t.co/DwtKQkSu #LIVEBLOG", "retweets": 30.0 }, +{ "user": "gadgetlab", "text": "#Apple #iPhone5 battery: 8hrs of 3G talk time, 8hrs of LTE or 3G browsing, 10hrs of WiFi browsing, 225hrs of standby http://t.co/DwtKQkSu", "retweets": 121.0 }, +{ "user": "gadgetlab", "text": "\"We've updated every aspect of iPhone 5,? @PSchiller says. #Apple #iPhone5 LIVEBLOG http://t.co/DwtKQkSu", "retweets": 16.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 display has 44% more color saturation. #Apple says its \"most accurate\" in industry http://t.co/DwtKQkSu LIVEBLOG by @redgirlsays", "retweets": 21.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 new A6 chip has 2x faster graphics and processing power http://t.co/DwtKQkSu LIVEBLOG by @redgirlsays", "retweets": 31.0 }, +{ "user": "gadgetlab", "text": "Yes, the #iPhone5 does have LTE. #Apple liveblog http://t.co/DwtKQkSu", "retweets": 19.0 }, +{ "user": "gadgetlab", "text": "#Apple #iPhone5 photos up on our liveblog ---> http://t.co/DwtKQkSu", "retweets": 35.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 has has a 326 PPI Retina display. The new screen is 4 inches, 1136x640 pixels w/ 16x9 aspect ratio http://t.co/DwtKQkSu #Apple", "retweets": 76.0 }, +{ "user": "gadgetlab", "text": "#iPhone5 is the thinnest & lightest phone Apple ever http://t.co/DwtKQkSu #Apple LIVEBLOG by @redgirlsays", "retweets": 46.0 }, +{ "user": "gadgetlab", "text": "Last month, Apple sold its 400 millionth iOS device. #iPhone5 LIVEBLOG http://t.co/DwtKQkSu by @redgirlsays", "retweets": 15.0 }, +{ "user": "gadgetlab", "text": "#Apple: More than 700,000 apps in App Store & 250,000 specifically for iPad http://t.co/DwtKQkSu #iPhone5 LIVEBLOG by @redgirlsays", "retweets": 6.0 }, +{ "user": "gadgetlab", "text": "#Apple CEO Tim Cook: 7million copies of Mountain Lion downloaded so far. Fastest selling version of OS X ever http://t.co/DwtKQkSu LIVEBLOG", "retweets": 10.0 }, +{ "user": "gadgetlab", "text": "Apple has 380 stores in 12 countries. On Friday it will open a store in its 13th, Sweden. http://t.co/5AtpHCfY #Apple #iPhone5 LIVEBLOG", "retweets": 11.0 }, +{ "user": "gadgetlab", "text": "#Apple CEO Tim Cook takes the stage at #iPhone5 event, has \"really cool stuff to show you.? http://t.co/DwtKQkSu liveblog by @redgirlsays", "retweets": 25.0 }, +{ "user": "gadgetlab", "text": "RT @wired: FYI, @wired's liveblog photos are Creative Commons - repost, if you like, with link and license http://t.co/PewWjvwX", "retweets": 26.0 }, +{ "user": "gadgetlab", "text": "Soon, photos from the inside of Apple's iPhone 5 event. @redgirlsays + @johnwbradley are on the scene... http://t.co/2mqmIhVB", "retweets": 14.0 }, +{ "user": "gadgetlab", "text": "Apple iPhone 5 event LIVEBLOG: the event starts in 30 mins but we've got some photos of reporters standing in line! http://t.co/2mqmIhVB", "retweets": 26.0 }, +{ "user": "gadgetlab", "text": "http://t.co/z7Dsi93S Search Confirms iPhone 5 with LTE Support, Plus New iPods http://t.co/a7mZnloy by @alexandra_chang", "retweets": 30.0 }, +{ "user": "gadgetlab", "text": "@redgirlsays on the scene at Apple's iPhone 5 unveiling & she has photos...of the outside of the event. Inside soon! http://t.co/5AtpHCfY", "retweets": 0.0 }, +{ "user": "gadgetlab", "text": "Demoing at an Apple Event: Secretive, Stressful, and (Probably) Worth It http://t.co/7NOIDSJw by @strngwys", "retweets": 11.0 } +]""" + val mashable = """[ +{ "user": "mashable", "text": "Which is sorriest, #Apple Maps or #Siri? @Charlie_White has the comic - http://t.co/YRXXJXGY", "retweets": 55.0 }, +{ "user": "mashable", "text": "Watch These Quadrotor #Robots Learn to Play Catch [VIDEO] http://t.co/CHZ5BPfL", "retweets": 21.0 }, +{ "user": "mashable", "text": "10 Funny @YouTube Clips of Sleeping Dogs http://t.co/ukPgWZjH", "retweets": 34.0 }, +{ "user": "mashable", "text": "65 Digital Media Resources You May Have Missed http://t.co/u1LJWc22", "retweets": 64.0 }, +{ "user": "mashable", "text": ".@HP Shows What @Windows 8 #Tablets for Business Will Look Like [PICS] http://t.co/Md6ua8Xc", "retweets": 32.0 }, +{ "user": "mashable", "text": "Twice as Many #Mobile News Readers Prefer Browsers to #Apps [STUDY] http://t.co/lEXhVLLJ", "retweets": 40.0 }, +{ "user": "mashable", "text": "7 Free #Android Apps for Killing Time in Lines http://t.co/eKu5hhsh", "retweets": 42.0 }, +{ "user": "mashable", "text": "Here's what you can expect at the Mashable #MediaSummit - http://t.co/S5Hxw3Iz Get your tickets today!", "retweets": 27.0 }, +{ "user": "mashable", "text": "'Dexter' Season 7 Premiere Available in Full on @YouTube [VIDEO] http://t.co/Cc5HeiZZ", "retweets": 345.0 }, +{ "user": "mashable", "text": ".@Amazon #Kindle Paperwhite is the Best E-Ink Screen You Can Buy [REVIEW] http://t.co/JDenefqi", "retweets": 55.0 }, +{ "user": "mashable", "text": "Watch These Epic #iPhone 5 Vs. #Android Music Videos http://t.co/Sf0De6de", "retweets": 49.0 }, +{ "user": "mashable", "text": "One Proposal to Cool a Warming Planet: An Umbrella Made of Asteroids http://t.co/ote1Dw3Z via @theatlantic", "retweets": 33.0 }, +{ "user": "mashable", "text": "Viral Video Recap: Bad Actors, #Apple Trolling and Slow-Motion Fireballs http://t.co/62XWSdWx", "retweets": 28.0 }, +{ "user": "mashable", "text": "#DidYouKnow 81% of mobile phone subscribers use text messaging? http://t.co/ABJMZpNZ", "retweets": 55.0 }, +{ "user": "mashable", "text": "Smartphone Camera Shootout: iPhone 5 vs. Galaxy SIII vs. iPhone 4S http://t.co/Qp0PM0fh", "retweets": 85.0 }, +{ "user": "mashable", "text": "Simon Cowell and http://t.co/sLfE3y62 Planning Show to Find the Next Steve Jobs http://t.co/zvIk0dwc", "retweets": 145.0 }, +{ "user": "mashable", "text": "Xi3 Packs a Fully Functional Desktop PC Into a Tiny Cube [VIDEO] http://t.co/OcOqZkU6", "retweets": 21.0 }, +{ "user": "mashable", "text": "How ?Mad Men? Could Turn You Into The Worst Employee Ever [INFOGRAPHIC] http://t.co/NngQ3qDT via @TheJaneDough", "retweets": 95.0 }, +{ "user": "mashable", "text": "Tumblr Will Be Down Next Weekend. Here's Why: http://t.co/xN8yFINP", "retweets": 129.0 }, +{ "user": "mashable", "text": "Top 10 Tech This Week http://t.co/N4emMtmD", "retweets": 50.0 }, +{ "user": "mashable", "text": "Which is Sorriest, Apple Maps or Siri? [SUNDAY COMICS] http://t.co/eka8gts0", "retweets": 89.0 }, +{ "user": "mashable", "text": "Book About Kickstarter Fails to Raise Enough Money on Kickstarter http://t.co/SpKYdnOx", "retweets": 59.0 }, +{ "user": "mashable", "text": "Top 10 #GIFs of the Week http://t.co/YVYDncNc", "retweets": 32.0 }, +{ "user": "mashable", "text": "Spookiest Smartphone Malware Yet? http://t.co/5Kk1DyPg via @TheAtlantic", "retweets": 57.0 }, +{ "user": "mashable", "text": "5 Mobile Photographers Capturing the World With #Android http://t.co/786NneBt", "retweets": 78.0 }, +{ "user": "mashable", "text": "Does This Video Show the iPad Mini? http://t.co/0Sbdpiu1", "retweets": 27.0 }, +{ "user": "mashable", "text": "Subscribe to Mashable on Kindle for the latest tech and social news on the go - http://t.co/HWWj3Xii", "retweets": 10.0 }, +{ "user": "mashable", "text": "Facebook to FTC: Let Us Advertise to Children http://t.co/mipwVjbp", "retweets": 28.0 }, +{ "user": "mashable", "text": "4 Tips to Avoid Work-From-Home Email Scams http://t.co/j4qxnmrj", "retweets": 43.0 }, +{ "user": "mashable", "text": "10 Amazing Google Earth and Maps Discoveries http://t.co/UyDMs135", "retweets": 144.0 }, +{ "user": "mashable", "text": "Top 5 Apps Your Kids Will Love This Week http://t.co/7YeW58Af", "retweets": 39.0 }, +{ "user": "mashable", "text": "#DidYouKnow T9 is short for \"Text on 9 keys\"? http://t.co/9rAFobkz", "retweets": 145.0 }, +{ "user": "mashable", "text": "Is your university better at social networking than you are? See how it ranks here: http://t.co/6aM1zpFa", "retweets": 57.0 }, +{ "user": "mashable", "text": "Facebook's upgraded its Messenger app for iPhone, and @ToddWasserman took it for a spin: http://t.co/MMgCEsJ2", "retweets": 47.0 }, +{ "user": "mashable", "text": "Imagine how it would feel to get fired from @Facebook. One man describes his experience. @joannellepan has the story -...", "retweets": 21.0 }, +{ "user": "mashable", "text": "Looking for a stand for your #iPad that doubles as a speaker? @Charlie_White found a great one: http://t.co/ImpCsgkd", "retweets": 49.0 }, +{ "user": "mashable", "text": "What would Psy's 'Gangnam Style' video be like without music? Even quirkier. @neeeda has the video - http://t.co/z6xrmclV", "retweets": 80.0 }, +{ "user": "mashable", "text": "Who knew there were such great games available for less than $15? @ChelseaBot, that's who: http://t.co/j5yqMbYd", "retweets": 24.0 }, +{ "user": "mashable", "text": "The 'Ostrich Pillow' claims to give you the ideal power-napping experience. @neeeda has more - http://t.co/YCzvanHH", "retweets": 77.0 }, +{ "user": "mashable", "text": "#DidYouKnow the Dvorak is a more efficient keyboard layout than QWERTY? http://t.co/IlHnfSsz", "retweets": 162.0 }, +{ "user": "mashable", "text": "Find out which fashion brands dominate social media - @ToddWasserman brings you this handy chart: http://t.co/hArNJOkE", "retweets": 49.0 }, +{ "user": "mashable", "text": "This iPad keyboard reveals what's wrong with Kickstarter. @PetePachal reviews it: http://t.co/2yjhGnY9", "retweets": 18.0 }, +{ "user": "mashable", "text": "Apple's given up on calling its Maps app the \"most powerful ever.\" @ToddWasserman explains why: http://t.co/3YatPmWf", "retweets": 98.0 }, +{ "user": "mashable", "text": "Week's worth of TV boiled down to the best 12 GIFs, thanks to the keen eye of @ChristErickson http://t.co/BAUiteqd", "retweets": 22.0 }, +{ "user": "mashable", "text": "Power Nap With This Head-Consuming Ostrich Pillow http://t.co/d34dzlPT", "retweets": 107.0 }, +{ "user": "mashable", "text": "#Bots Drive 16% of U.S. Web Traffic [INFOGRAPHIC] http://t.co/lfFAjUy6", "retweets": 84.0 }, +{ "user": "mashable", "text": "#Apple Cites Improved Working Conditions in Foreign Factories [REPORT] http://t.co/PKmkIYYK", "retweets": 22.0 }, +{ "user": "mashable", "text": "Bizarre @YouTube Video Removes Music From 'Gangnam Style' http://t.co/AzSLo2cq", "retweets": 47.0 }, +{ "user": "mashable", "text": "These #Robots Can Help Disabled Officers, Veterans Get Back to Work [VIDEO] http://t.co/wg29VIx7", "retweets": 28.0 }, +{ "user": "mashable", "text": "Like viral videos and gadget demos? Subscribe to the Mashable YouTube channel - http://t.co/Sp0NWvgo", "retweets": 23.0 }, +{ "user": "mashable", "text": "Biodegradable Medical Implants Dissolve in Water [VIDEO] http://t.co/XVseXUaa", "retweets": 36.0 }, +{ "user": "mashable", "text": "5 Easy Steps To Make Your Job Descriptions Go Viral http://t.co/a5ArxrL5", "retweets": 56.0 }, +{ "user": "mashable", "text": "Why Social Media Makes Customer Service Better http://t.co/JkdkzqnV", "retweets": 167.0 }, +{ "user": "mashable", "text": "10 Essential Resources for Bootstrapping #Businesses http://t.co/1e3hqvZA", "retweets": 39.0 }, +{ "user": "mashable", "text": "The Digerati: 7 Profiles of Successful #Entrepreneurs http://t.co/M5l5CmNF", "retweets": 25.0 }, +{ "user": "mashable", "text": "10 Amazing @Google Earth and Maps Discoveries http://t.co/gDrlBXJ8", "retweets": 115.0 }, +{ "user": "mashable", "text": "Former Employee Explains Why He Was Fired From @Facebook http://t.co/3n1wb5fJ", "retweets": 56.0 }, +{ "user": "mashable", "text": "6 #Apps You Don't Want to Miss http://t.co/cQxIq9VU", "retweets": 78.0 }, +{ "user": "mashable", "text": "#DidYouKnow the QWERTY keyboard layout was invented in 1868 -- and it's proven to be inefficient? http://t.co/5q3kKrQy", "retweets": 152.0 }, +{ "user": "mashable", "text": "Which Are the Most Social Colleges? [INFOGRAPHIC] http://t.co/8eztys5v", "retweets": 60.0 }, +{ "user": "mashable", "text": "Facebook Embraces Bubbles in Messenger iPhone Upgrade http://t.co/NaFHao22", "retweets": 61.0 }, +{ "user": "mashable", "text": "Zooka Speaker Bar Makes Any Bluetooth Device Sound Better [REVIEW] http://t.co/LN6jWldu", "retweets": 23.0 }, +{ "user": "mashable", "text": "Top Comments From Mashable Readers This Week http://t.co/E0ANQg3q", "retweets": 18.0 }, +{ "user": "mashable", "text": "FinderCodes Uses QR Codes to Find Lost Phones http://t.co/kn3euvFy", "retweets": 28.0 }, +{ "user": "mashable", "text": "5 Games You Should Play This Weekend http://t.co/t0kxXONY", "retweets": 42.0 }, +{ "user": "mashable", "text": "How Much Bandwidth Will the Vice Presidential Debate Need? http://t.co/ysANGsVP", "retweets": 33.0 }, +{ "user": "mashable", "text": "#DidYouKnow the JPG is a compression technique, not a file size? http://t.co/bVJjxUFJ", "retweets": 117.0 }, +{ "user": "mashable", "text": "Something Is Missing, Batman [COMIC] http://t.co/TBlNdfBg", "retweets": 20.0 }, +{ "user": "mashable", "text": "Burberry Tops Fashion Brands In Social Media This Week [CHART] http://t.co/8GZl4U5t", "retweets": 64.0 }, +{ "user": "mashable", "text": "Video Callers Know No Bounds ? Not Even Bathrooms http://t.co/JmNNbAkZ via @BNDarticles", "retweets": 14.0 }, +{ "user": "mashable", "text": "CruxSkunk iPad Keyboard Exposes the Mirage of Kickstarter [REVIEW] http://t.co/R7jjva6V", "retweets": 13.0 }, +{ "user": "mashable", "text": "Apple No Longer Saying Maps Are 'Most Powerful Ever' http://t.co/MhofMrC7", "retweets": 100.0 }, +{ "user": "mashable", "text": "Doing some weekend reading? Check out the new and improved Mashable iPad app for our latest news - http://t.co/G1S46WJe", "retweets": 11.0 }, +{ "user": "mashable", "text": "Parking Panda App Finds and Guarantees Parking Spots http://t.co/D9kl2Hsc", "retweets": 41.0 }, +{ "user": "mashable", "text": "This Week's Top TV Moments in GIF Form http://t.co/V2asPRBp", "retweets": 26.0 }, +{ "user": "mashable", "text": "Small Business Advertisers Like Facebook's Immediacy, But Not Its Metrics http://t.co/KPYD1UlC", "retweets": 50.0 }, +{ "user": "mashable", "text": "iOS 6 Users Complain About Wi-Fi, Connectivity Issues - http://t.co/iogRstNn", "retweets": 180.0 }, +{ "user": "mashable", "text": "#DidYouKnow the 1962 'Sensorama Stimulator' was a precursor to augmented reality?http://t.co/0t4Eqjuq", "retweets": 59.0 }, +{ "user": "mashable", "text": "Our iPhones Are Depleting the Earth's Resources [INFOGRAPHIC] http://t.co/XnTLqe0p", "retweets": 205.0 }, +{ "user": "mashable", "text": "iPad 4 Has Carbon Fiber Body, Flexible Display [REPORT] http://t.co/Dft5VoXD via @tabtimes", "retweets": 198.0 }, +{ "user": "mashable", "text": "Disagreement over Voice Guided Turn-by-Turn Convinced Apple to Develop own Map App http://t.co/tMwL9IVN", "retweets": 49.0 }, +{ "user": "mashable", "text": "Derrick Rose's Brilliant Digital-Only Marketing Campaign #TheReturn http://t.co/S2lfS9Sw", "retweets": 73.0 }, +{ "user": "mashable", "text": "New Rolling Stone Cover Features YouTube-Grown duo Garmin http://t.co/gxug1n8F", "retweets": 18.0 }, +{ "user": "mashable", "text": "Facebook and the Gates Foundation Organize Hackathon to Develop Apps for Future College Students http://t.co/7tvGDMCe", "retweets": 80.0 }, +{ "user": "mashable", "text": "#DidYouKnow Sennheiser made a $40,000 pair of headphones? http://t.co/JbOWGElL", "retweets": 71.0 }, +{ "user": "mashable", "text": "Only 10% of Tweets About Apple Maps Are Positive [REPORT] http://t.co/wmGh8pGx", "retweets": 116.0 }, +{ "user": "mashable", "text": "Justin Bieber's Fan Tribute Gets 200,000 Retweets http://t.co/gWtxdPHa", "retweets": 14.0 }, +{ "user": "mashable", "text": "These Are the New Tech Job Hot Spots [INFOGRAPHIC] http://t.co/w1e2AHMp", "retweets": 79.0 }, +{ "user": "mashable", "text": "82 tech gadgets and products that are worth the splurge - http://t.co/HaNdfLRW #TheHotList", "retweets": 98.0 }, +{ "user": "mashable", "text": "Startup Marketplace Connects Fashion Brands With Boutiques http://t.co/IQ0rA7HI", "retweets": 27.0 }, +{ "user": "mashable", "text": "Time Inc. CEO Promises Less Free Content http://t.co/Ftb2w4me", "retweets": 20.0 }, +{ "user": "mashable", "text": "You can now read the latest from Mashable on @pulsepad - http://t.co/h3d6lULz", "retweets": 15.0 }, +{ "user": "mashable", "text": "Many Windows 8 Tablets Will Sport a Keyboard http://t.co/5OpFx5VR via: @techreview", "retweets": 33.0 }, +{ "user": "mashable", "text": "The Bluth Family Gets Political in New Tumblr Page http://t.co/t4OzctO0", "retweets": 16.0 }, +{ "user": "mashable", "text": "China's Internet Is Getting Faster [INFOGRAPHIC] http://t.co/4dqj8Ln0", "retweets": 63.0 }, +{ "user": "mashable", "text": "Camera+ Arrives on the iPad http://t.co/EG4nDhsQ", "retweets": 25.0 }, +{ "user": "mashable", "text": "Cute Images Make Us 44% Sharper [STUDY] http://t.co/I4IiIbHX", "retweets": 42.0 }, +{ "user": "mashable", "text": "Take 100 NASA Photos, Stir, Make Van Gogh's 'Starry Night' http://t.co/dQv6mvwI", "retweets": 73.0 }, +{ "user": "mashable", "text": "Why Being Unfriended on Facebook Hurts [STUDY] http://t.co/0eHHUrTo", "retweets": 51.0 }, +{ "user": "mashable", "text": "Think Apple Maps Is Bad? Remember the Tale of MobileMe http://t.co/Koeew9eU", "retweets": 41.0 } +]""" +} diff --git a/Scala/objsets/src/main/scala/objsets/TweetReader.scala b/Scala/objsets/src/main/scala/objsets/TweetReader.scala new file mode 100644 index 0000000..289e6ed --- /dev/null +++ b/Scala/objsets/src/main/scala/objsets/TweetReader.scala @@ -0,0 +1,78 @@ +package objsets + +object TweetReader { + + object ParseTweets { + import scala.util.parsing.json._ + + def getList[T](s: String): List[T] = + JSON.parseFull(s).get.asInstanceOf[List[T]] + + def getMap(s: String): Map[String, Any] = + JSON.parseFull(s).get.asInstanceOf[Map[String, Any]] + + def getTweets(user: String, json: String): List[Tweet] = + for (map <- getList[Map[String, Any]](json)) yield { + val text = map("text") + val retweets = map("retweet_count") + new Tweet(user, text.toString, retweets.toString.toDouble.toInt) + } + + def getTweetData(user: String, json: String): List[Tweet] = { + // is list + val l = getList[Map[String, Any]](json) + for (map <- l) yield { + val text = map("text") + val retweets = map("retweets") + new Tweet(user, text.toString, retweets.toString.toDouble.toInt) + } + } + } + + def toTweetSet(l: List[Tweet]): TweetSet = { + l.foldLeft(new Empty: TweetSet)(_.incl(_)) + } + + def unparseToData(tws: List[Tweet]): String = { + val buf = new StringBuffer + for (tw <- tws) { + val json = "{ \"user\": \"" + tw.user + "\", \"text\": \"" + + tw.text.replaceAll(""""""", "\\\\\\\"") + "\", \"retweets\": " + + tw.retweets + ".0 }" + buf.append(json + ",\n") + } + buf.toString + } + + val sites = List("gizmodo", "TechCrunch", "engadget", "amazondeals", "CNET", "gadgetlab", "mashable") + + private val gizmodoTweets = TweetReader.ParseTweets.getTweetData("gizmodo", TweetData.gizmodo) + private val techCrunchTweets = TweetReader.ParseTweets.getTweetData("TechCrunch", TweetData.TechCrunch) + private val engadgetTweets = TweetReader.ParseTweets.getTweetData("engadget", TweetData.engadget) + private val amazondealsTweets = TweetReader.ParseTweets.getTweetData("amazondeals", TweetData.amazondeals) + private val cnetTweets = TweetReader.ParseTweets.getTweetData("CNET", TweetData.CNET) + private val gadgetlabTweets = TweetReader.ParseTweets.getTweetData("gadgetlab", TweetData.gadgetlab) + private val mashableTweets = TweetReader.ParseTweets.getTweetData("mashable", TweetData.mashable) + + private val sources = List(gizmodoTweets, techCrunchTweets, engadgetTweets, amazondealsTweets, cnetTweets, gadgetlabTweets, mashableTweets) + + val tweetMap: Map[String, List[Tweet]] = + Map() ++ Seq((sites(0) -> gizmodoTweets), + (sites(1) -> techCrunchTweets), + (sites(2) -> engadgetTweets), + (sites(3) -> amazondealsTweets), + (sites(4) -> cnetTweets), + (sites(5) -> gadgetlabTweets), + (sites(6) -> mashableTweets)) + + val tweetSets: List[TweetSet] = sources.map(tweets => toTweetSet(tweets)) + + private val siteTweetSetMap: Map[String, TweetSet] = + Map() ++ (sites zip tweetSets) + + private def unionOfAllTweetSets(curSets: List[TweetSet], acc: TweetSet): TweetSet = + if (curSets.isEmpty) acc + else unionOfAllTweetSets(curSets.tail, acc.union(curSets.head)) + + val allTweets: TweetSet = unionOfAllTweetSets(tweetSets, new Empty) +} diff --git a/Scala/objsets/src/main/scala/objsets/TweetSet.scala b/Scala/objsets/src/main/scala/objsets/TweetSet.scala new file mode 100644 index 0000000..9200c50 --- /dev/null +++ b/Scala/objsets/src/main/scala/objsets/TweetSet.scala @@ -0,0 +1,200 @@ +package objsets + +import common._ +import TweetReader._ + +/** + * A class to represent tweets. + */ +class Tweet(val user: String, val text: String, val retweets: Int) { + override def toString: String = + "User: " + user + "\n" + + "Text: " + text + " [" + retweets + "]" +} + +/** + * This represents a set of objects of type `Tweet` in the form of a binary search + * tree. Every branch in the tree has two children (two `TweetSet`s). There is an + * invariant which always holds: for every branch `b`, all elements in the left + * subtree are smaller than the tweet at `b`. The eleemnts in the right subtree are + * larger. + * + * Note that the above structure requires us to be able to compare two tweets (we + * need to be able to say which of two tweets is larger, or if they are equal). In + * this implementation, the equality / order of tweets is based on the tweet's text + * (see `def incl`). Hence, a `TweetSet` could not contain two tweets with the same + * text from different users. + * + * + * The advantage of representing sets as binary search trees is that the elements + * of the set can be found quickly. If you want to learn more you can take a look + * at the Wikipedia page [1], but this is not necessary in order to solve this + * assignment. + * + * [1] http://en.wikipedia.org/wiki/Binary_search_tree + */ +abstract class TweetSet { + + /** + * This method takes a predicate and returns a subset of all the elements + * in the original set for which the predicate is true. + * + * Question: Can we implment this method here, or should it remain abstract + * and be implemented in the subclasses? + */ + def filter(p: Tweet => Boolean): TweetSet = ??? + + /** + * This is a helper method for `filter` that propagetes the accumulated tweets. + */ + def filterAcc(p: Tweet => Boolean, acc: TweetSet): TweetSet + + /** + * Returns a new `TweetSet` that is the union of `TweetSet`s `this` and `that`. + * + * Question: Should we implment this method here, or should it remain abstract + * and be implemented in the subclasses? + */ + def union(that: TweetSet): TweetSet = ??? + + /** + * Returns the tweet from this set which has the greatest retweet count. + * + * Calling `mostRetweeted` on an empty set should throw an exception of + * type `java.util.NoSuchElementException`. + * + * Question: Should we implment this method here, or should it remain abstract + * and be implemented in the subclasses? + */ + def mostRetweeted: Tweet = ??? + + /** + * Returns a list containing all tweets of this set, sorted by retweet count + * in descending order. In other words, the head of the resulting list should + * have the highest retweet count. + * + * Hint: the method `remove` on TweetSet will be very useful. + * Question: Should we implment this method here, or should it remain abstract + * and be implemented in the subclasses? + */ + def descendingByRetweet: TweetList = ??? + + + /** + * The following methods are already implemented + */ + + /** + * Returns a new `TweetSet` which contains all elements of this set, and the + * the new element `tweet` in case it does not already exist in this set. + * + * If `this.contains(tweet)`, the current set is returned. + */ + def incl(tweet: Tweet): TweetSet + + /** + * Returns a new `TweetSet` which excludes `tweet`. + */ + def remove(tweet: Tweet): TweetSet + + /** + * Tests if `tweet` exists in this `TweetSet`. + */ + def contains(tweet: Tweet): Boolean + + /** + * This method takes a function and applies it to every element in the set. + */ + def foreach(f: Tweet => Unit): Unit +} + +class Empty extends TweetSet { + + def filterAcc(p: Tweet => Boolean, acc: TweetSet): TweetSet = ??? + + + /** + * The following methods are already implemented + */ + + def contains(tweet: Tweet): Boolean = false + + def incl(tweet: Tweet): TweetSet = new NonEmpty(tweet, new Empty, new Empty) + + def remove(tweet: Tweet): TweetSet = this + + def foreach(f: Tweet => Unit): Unit = () +} + +class NonEmpty(elem: Tweet, left: TweetSet, right: TweetSet) extends TweetSet { + + def filterAcc(p: Tweet => Boolean, acc: TweetSet): TweetSet = ??? + + + /** + * The following methods are already implemented + */ + + def contains(x: Tweet): Boolean = + if (x.text < elem.text) left.contains(x) + else if (elem.text < x.text) right.contains(x) + else true + + def incl(x: Tweet): TweetSet = { + if (x.text < elem.text) new NonEmpty(elem, left.incl(x), right) + else if (elem.text < x.text) new NonEmpty(elem, left, right.incl(x)) + else this + } + + def remove(tw: Tweet): TweetSet = + if (tw.text < elem.text) new NonEmpty(elem, left.remove(tw), right) + else if (elem.text < tw.text) new NonEmpty(elem, left, right.remove(tw)) + else left.union(right) + + def foreach(f: Tweet => Unit): Unit = { + f(elem) + left.foreach(f) + right.foreach(f) + } +} + +trait TweetList { + def head: Tweet + def tail: TweetList + def isEmpty: Boolean + def foreach(f: Tweet => Unit): Unit = + if (!isEmpty) { + f(head) + tail.foreach(f) + } +} + +object Nil extends TweetList { + def head = throw new java.util.NoSuchElementException("head of EmptyList") + def tail = throw new java.util.NoSuchElementException("tail of EmptyList") + def isEmpty = true +} + +class Cons(val head: Tweet, val tail: TweetList) extends TweetList { + def isEmpty = false +} + + +object GoogleVsApple { + val google = List("android", "Android", "galaxy", "Galaxy", "nexus", "Nexus") + val apple = List("ios", "iOS", "iphone", "iPhone", "ipad", "iPad") + + lazy val googleTweets: TweetSet = ??? + lazy val appleTweets: TweetSet = ??? + + /** + * A list of all tweets mentioning a keyword from either apple or google, + * sorted by the number of retweets. + */ + lazy val trending: TweetList = ??? +} + +object Main extends App { + // Print the trending tweets + GoogleVsApple.trending foreach println +} diff --git a/Scala/objsets/src/test/scala/objsets/TweetSetSuite.scala b/Scala/objsets/src/test/scala/objsets/TweetSetSuite.scala new file mode 100644 index 0000000..be82938 --- /dev/null +++ b/Scala/objsets/src/test/scala/objsets/TweetSetSuite.scala @@ -0,0 +1,72 @@ +package objsets + +import org.scalatest.FunSuite + +import org.junit.runner.RunWith +import org.scalatest.junit.JUnitRunner + +@RunWith(classOf[JUnitRunner]) +class TweetSetSuite extends FunSuite { + trait TestSets { + val set1 = new Empty + val set2 = set1.incl(new Tweet("a", "a body", 20)) + val set3 = set2.incl(new Tweet("b", "b body", 20)) + val c = new Tweet("c", "c body", 7) + val d = new Tweet("d", "d body", 9) + val set4c = set3.incl(c) + val set4d = set3.incl(d) + val set5 = set4c.incl(d) + } + + def asSet(tweets: TweetSet): Set[Tweet] = { + var res = Set[Tweet]() + tweets.foreach(res += _) + res + } + + def size(set: TweetSet): Int = asSet(set).size + + test("filter: on empty set") { + new TestSets { + assert(size(set1.filter(tw => tw.user == "a")) === 0) + } + } + + test("filter: a on set5") { + new TestSets { + assert(size(set5.filter(tw => tw.user == "a")) === 1) + } + } + + test("filter: 20 on set5") { + new TestSets { + assert(size(set5.filter(tw => tw.retweets == 20)) === 2) + } + } + + test("union: set4c and set4d") { + new TestSets { + assert(size(set4c.union(set4d)) === 4) + } + } + + test("union: with empty set (1)") { + new TestSets { + assert(size(set5.union(set1)) === 4) + } + } + + test("union: with empty set (2)") { + new TestSets { + assert(size(set1.union(set5)) === 4) + } + } + + test("descending: set5") { + new TestSets { + val trends = set5.descendingByRetweet + assert(!trends.isEmpty) + assert(trends.head.user == "a" || trends.head.user == "b") + } + } +} |