[{"data":1,"prerenderedAt":1427},["ShallowReactive",2],{"blog:2015:time-window-events-with-apache-spark-streaming":3,"blogMore-Development":1392,"comments-time-window-events-with-apache-spark-streaming":1405},{"_path":4,"_dir":5,"_draft":6,"_partial":6,"_locale":7,"title":8,"description":9,"date":10,"category":11,"tags":12,"excerpt":15,"body":30,"_type":1384,"_id":1385,"_source":1386,"_file":1387,"_stem":1388,"_extension":1389,"url":1390,"wordCount":1391,"minutes":139,"commentCount":73},"/blog/2015/time-window-events-with-apache-spark-streaming","2015",false,"en","Time window events with Apache Spark Streaming","If you’re working with Spark Streaming, you might run into an interesting problem if you want to output an event based on multiple messages within a specific time period.","2015-06-27T12:46:25+00:00","Development",[13,14],"Apache Spark","Scala",{"type":16,"children":17},"root",[18,25],{"type":19,"tag":20,"props":21,"children":22},"element","p",{},[23],{"type":24,"value":9},"text",{"type":19,"tag":20,"props":26,"children":27},{},[28],{"type":24,"value":29},"For example, I want to send a security alert if I see 10 DDOS attempts to an IP address in a five-minute window.",{"type":16,"children":31,"toc":1378},[32,36,40,47,58,405,411,424,429,444,449,455,460,482,503,628,640,653,669,675,687,1358,1363,1372],{"type":19,"tag":20,"props":33,"children":34},{},[35],{"type":24,"value":9},{"type":19,"tag":20,"props":37,"children":38},{},[39],{"type":24,"value":29},{"type":19,"tag":41,"props":42,"children":44},"h2",{"id":43},"groupbykeyandwindow",[45],{"type":24,"value":46},"groupByKeyAndWindow",{"type":19,"tag":20,"props":48,"children":49},{},[50,56],{"type":19,"tag":51,"props":52,"children":54},"code",{"className":53},[],[55],{"type":24,"value":46},{"type":24,"value":57}," allows us to choose the IP address for the key and 5 minutes for the window. If we wanted to subsequently collect the sourceIp and the timestamp, it looks like this:",{"type":19,"tag":59,"props":60,"children":65},"pre",{"className":61,"code":62,"language":63,"meta":64,"style":64},"language-scala shiki shiki-themes everforest-light dracula","var messageLimit = 10\nvar messageWindow = Minutes(5)\nval scc = new StreamingContext(conf, Minutes(1))\n\n// ... setup Kafka consumer via SparkUtils\nkafkaConsumer\n    .flatMap(parseSecurityMessage)\n    .filter(m => m.securityType == 'DDOS')\n    .map(m => m.targetIp -> Seq((m.timestamp, m.sourceIp)))\n    .reduceByKeyAndWindow({(x, y) => x ++ y}, messageWindow)\n    .filter(g => g._2.length >= messageLimit)\n    .foreachRDD(m => m.foreach(createAlertEvent))\n\nscc.start()\nscc.streamUntilTerminated()\n","scala","",[66],{"type":19,"tag":51,"props":67,"children":68},{"__ignoreMap":64},[69,99,137,189,199,209,218,227,272,305,333,361,379,387,396],{"type":19,"tag":70,"props":71,"children":74},"span",{"class":72,"line":73},"line",1,[75,81,87,93],{"type":19,"tag":70,"props":76,"children":78},{"style":77},"--shiki-default:#F85552;--shiki-dark:#FF79C6",[79],{"type":24,"value":80},"var",{"type":19,"tag":70,"props":82,"children":84},{"style":83},"--shiki-default:#5C6A72;--shiki-dark:#F8F8F2",[85],{"type":24,"value":86}," messageLimit ",{"type":19,"tag":70,"props":88,"children":90},{"style":89},"--shiki-default:#F57D26;--shiki-dark:#FF79C6",[91],{"type":24,"value":92},"=",{"type":19,"tag":70,"props":94,"children":96},{"style":95},"--shiki-default:#DF69BA;--shiki-dark:#BD93F9",[97],{"type":24,"value":98}," 10\n",{"type":19,"tag":70,"props":100,"children":102},{"class":72,"line":101},2,[103,107,112,116,122,127,132],{"type":19,"tag":70,"props":104,"children":105},{"style":77},[106],{"type":24,"value":80},{"type":19,"tag":70,"props":108,"children":109},{"style":83},[110],{"type":24,"value":111}," messageWindow ",{"type":19,"tag":70,"props":113,"children":114},{"style":89},[115],{"type":24,"value":92},{"type":19,"tag":70,"props":117,"children":119},{"style":118},"--shiki-default:#DFA000;--shiki-dark:#8BE9FD",[120],{"type":24,"value":121}," Minutes",{"type":19,"tag":70,"props":123,"children":124},{"style":83},[125],{"type":24,"value":126},"(",{"type":19,"tag":70,"props":128,"children":129},{"style":95},[130],{"type":24,"value":131},"5",{"type":19,"tag":70,"props":133,"children":134},{"style":83},[135],{"type":24,"value":136},")\n",{"type":19,"tag":70,"props":138,"children":140},{"class":72,"line":139},3,[141,146,151,155,160,165,170,175,179,184],{"type":19,"tag":70,"props":142,"children":143},{"style":89},[144],{"type":24,"value":145},"val",{"type":19,"tag":70,"props":147,"children":148},{"style":83},[149],{"type":24,"value":150}," scc ",{"type":19,"tag":70,"props":152,"children":153},{"style":89},[154],{"type":24,"value":92},{"type":19,"tag":70,"props":156,"children":157},{"style":77},[158],{"type":24,"value":159}," new",{"type":19,"tag":70,"props":161,"children":162},{"style":118},[163],{"type":24,"value":164}," StreamingContext",{"type":19,"tag":70,"props":166,"children":167},{"style":83},[168],{"type":24,"value":169},"(conf, ",{"type":19,"tag":70,"props":171,"children":172},{"style":118},[173],{"type":24,"value":174},"Minutes",{"type":19,"tag":70,"props":176,"children":177},{"style":83},[178],{"type":24,"value":126},{"type":19,"tag":70,"props":180,"children":181},{"style":95},[182],{"type":24,"value":183},"1",{"type":19,"tag":70,"props":185,"children":186},{"style":83},[187],{"type":24,"value":188},"))\n",{"type":19,"tag":70,"props":190,"children":192},{"class":72,"line":191},4,[193],{"type":19,"tag":70,"props":194,"children":196},{"emptyLinePlaceholder":195},true,[197],{"type":24,"value":198},"\n",{"type":19,"tag":70,"props":200,"children":202},{"class":72,"line":201},5,[203],{"type":19,"tag":70,"props":204,"children":206},{"style":205},"--shiki-default:#939F91;--shiki-default-font-style:italic;--shiki-dark:#6272A4;--shiki-dark-font-style:inherit",[207],{"type":24,"value":208},"// ... setup Kafka consumer via SparkUtils\n",{"type":19,"tag":70,"props":210,"children":212},{"class":72,"line":211},6,[213],{"type":19,"tag":70,"props":214,"children":215},{"style":83},[216],{"type":24,"value":217},"kafkaConsumer\n",{"type":19,"tag":70,"props":219,"children":221},{"class":72,"line":220},7,[222],{"type":19,"tag":70,"props":223,"children":224},{"style":83},[225],{"type":24,"value":226},"    .flatMap(parseSecurityMessage)\n",{"type":19,"tag":70,"props":228,"children":230},{"class":72,"line":229},8,[231,236,241,246,251,257,263,268],{"type":19,"tag":70,"props":232,"children":233},{"style":83},[234],{"type":24,"value":235},"    .filter(m ",{"type":19,"tag":70,"props":237,"children":238},{"style":89},[239],{"type":24,"value":240},"=>",{"type":19,"tag":70,"props":242,"children":243},{"style":83},[244],{"type":24,"value":245}," m.securityType ",{"type":19,"tag":70,"props":247,"children":248},{"style":89},[249],{"type":24,"value":250},"==",{"type":19,"tag":70,"props":252,"children":254},{"style":253},"--shiki-default:#5C6A72;--shiki-dark:#BD93F9",[255],{"type":24,"value":256}," '",{"type":19,"tag":70,"props":258,"children":260},{"style":259},"--shiki-default:#DFA000;--shiki-default-font-style:inherit;--shiki-default-text-decoration:inherit;--shiki-dark:#FF5555;--shiki-dark-font-style:italic;--shiki-dark-text-decoration:underline",[261],{"type":24,"value":262},"DDOS",{"type":19,"tag":70,"props":264,"children":265},{"style":253},[266],{"type":24,"value":267},"'",{"type":19,"tag":70,"props":269,"children":270},{"style":83},[271],{"type":24,"value":136},{"type":19,"tag":70,"props":273,"children":275},{"class":72,"line":274},9,[276,281,285,290,295,300],{"type":19,"tag":70,"props":277,"children":278},{"style":83},[279],{"type":24,"value":280},"    .map(m ",{"type":19,"tag":70,"props":282,"children":283},{"style":89},[284],{"type":24,"value":240},{"type":19,"tag":70,"props":286,"children":287},{"style":83},[288],{"type":24,"value":289}," m.targetIp ",{"type":19,"tag":70,"props":291,"children":292},{"style":89},[293],{"type":24,"value":294},"->",{"type":19,"tag":70,"props":296,"children":297},{"style":118},[298],{"type":24,"value":299}," Seq",{"type":19,"tag":70,"props":301,"children":302},{"style":83},[303],{"type":24,"value":304},"((m.timestamp, m.sourceIp)))\n",{"type":19,"tag":70,"props":306,"children":308},{"class":72,"line":307},10,[309,314,318,323,328],{"type":19,"tag":70,"props":310,"children":311},{"style":83},[312],{"type":24,"value":313},"    .reduceByKeyAndWindow({(x, y) ",{"type":19,"tag":70,"props":315,"children":316},{"style":89},[317],{"type":24,"value":240},{"type":19,"tag":70,"props":319,"children":320},{"style":83},[321],{"type":24,"value":322}," x ",{"type":19,"tag":70,"props":324,"children":325},{"style":89},[326],{"type":24,"value":327},"++",{"type":19,"tag":70,"props":329,"children":330},{"style":83},[331],{"type":24,"value":332}," y}, messageWindow)\n",{"type":19,"tag":70,"props":334,"children":336},{"class":72,"line":335},11,[337,342,346,351,356],{"type":19,"tag":70,"props":338,"children":339},{"style":83},[340],{"type":24,"value":341},"    .filter(g ",{"type":19,"tag":70,"props":343,"children":344},{"style":89},[345],{"type":24,"value":240},{"type":19,"tag":70,"props":347,"children":348},{"style":83},[349],{"type":24,"value":350}," g._2.length ",{"type":19,"tag":70,"props":352,"children":353},{"style":89},[354],{"type":24,"value":355},">=",{"type":19,"tag":70,"props":357,"children":358},{"style":83},[359],{"type":24,"value":360}," messageLimit)\n",{"type":19,"tag":70,"props":362,"children":364},{"class":72,"line":363},12,[365,370,374],{"type":19,"tag":70,"props":366,"children":367},{"style":83},[368],{"type":24,"value":369},"    .foreachRDD(m ",{"type":19,"tag":70,"props":371,"children":372},{"style":89},[373],{"type":24,"value":240},{"type":19,"tag":70,"props":375,"children":376},{"style":83},[377],{"type":24,"value":378}," m.foreach(createAlertEvent))\n",{"type":19,"tag":70,"props":380,"children":382},{"class":72,"line":381},13,[383],{"type":19,"tag":70,"props":384,"children":385},{"emptyLinePlaceholder":195},[386],{"type":24,"value":198},{"type":19,"tag":70,"props":388,"children":390},{"class":72,"line":389},14,[391],{"type":19,"tag":70,"props":392,"children":393},{"style":83},[394],{"type":24,"value":395},"scc.start()\n",{"type":19,"tag":70,"props":397,"children":399},{"class":72,"line":398},15,[400],{"type":19,"tag":70,"props":401,"children":402},{"style":83},[403],{"type":24,"value":404},"scc.streamUntilTerminated()\n",{"type":19,"tag":41,"props":406,"children":408},{"id":407},"problem",[409],{"type":24,"value":410},"Problem",{"type":19,"tag":20,"props":412,"children":413},{},[414,416,422],{"type":24,"value":415},"The problem is ",{"type":19,"tag":417,"props":418,"children":419},"strong",{},[420],{"type":24,"value":421},"your event fires many times as the stateless RDD is re-run every batch period",{"type":24,"value":423},".",{"type":19,"tag":20,"props":425,"children":426},{},[427],{"type":24,"value":428},"The simplest solution would be to make the batch interval the same as your message window size, but that causes more problems, namely:",{"type":19,"tag":430,"props":431,"children":432},"ul",{},[433,439],{"type":19,"tag":434,"props":435,"children":436},"li",{},[437],{"type":24,"value":438},"Your job can’t perform any other triggers on the source data at a shorter interval",{"type":19,"tag":434,"props":440,"children":441},{},[442],{"type":24,"value":443},"You won’t know about these alerts until some time after they happen (in this case 5 minutes)",{"type":19,"tag":20,"props":445,"children":446},{},[447],{"type":24,"value":448},"External would be terrible, and neither Spark counters nor globals are much use here.",{"type":19,"tag":41,"props":450,"children":452},{"id":451},"solution",[453],{"type":24,"value":454},"Solution",{"type":19,"tag":20,"props":456,"children":457},{},[458],{"type":24,"value":459},"We need to do two things:",{"type":19,"tag":461,"props":462,"children":463},"ol",{},[464,477],{"type":19,"tag":434,"props":465,"children":466},{},[467,469,475],{"type":24,"value":468},"Stop the RDD re-running and instead use the streaming state. We can do this by using the ",{"type":19,"tag":51,"props":470,"children":472},{"className":471},[],[473],{"type":24,"value":474},"reduceByKeyAndWindow",{"type":24,"value":476}," overload that allows us to specify the inverse function for removing data as it goes out of the window.",{"type":19,"tag":434,"props":478,"children":479},{},[480],{"type":24,"value":481},"Introduce a small amount of in-RDD state used to identify when the event is clear and when it should fire again.",{"type":19,"tag":20,"props":483,"children":484},{},[485,487,493,495,501],{"type":24,"value":486},"Let us assume we have a class to handle part 2 named ",{"type":19,"tag":51,"props":488,"children":490},{"className":489},[],[491],{"type":24,"value":492},"WindowEventTrigger",{"type":24,"value":494}," that provides add and remove methods and a boolean ",{"type":19,"tag":51,"props":496,"children":498},{"className":497},[],[499],{"type":24,"value":500},"triggerNow",{"type":24,"value":502}," flag that identifies when the event should re-fire. Our RDD body would now look like this:",{"type":19,"tag":59,"props":504,"children":506},{"className":61,"code":505,"language":63,"meta":64,"style":64},"kafkaConsumer\n    .flatMap(parseSecurityMessage)\n    .filter(m => m.securityType == 'DDOS')\n    .map(m => m.targetIp -> WindowEventTrigger(Seq(m.timestamp, m.sourceIp), messageLimit))\n    .reduceByKeyAndWindow(_ add _, _ remove _, messageWindow)\n    .filter(_._2.triggerNow)\n    .foreachRDD(m => m.foreach(createAlertEvent))\n",[507],{"type":19,"tag":51,"props":508,"children":509},{"__ignoreMap":64},[510,517,524,559,597,605,613],{"type":19,"tag":70,"props":511,"children":512},{"class":72,"line":73},[513],{"type":19,"tag":70,"props":514,"children":515},{"style":83},[516],{"type":24,"value":217},{"type":19,"tag":70,"props":518,"children":519},{"class":72,"line":101},[520],{"type":19,"tag":70,"props":521,"children":522},{"style":83},[523],{"type":24,"value":226},{"type":19,"tag":70,"props":525,"children":526},{"class":72,"line":139},[527,531,535,539,543,547,551,555],{"type":19,"tag":70,"props":528,"children":529},{"style":83},[530],{"type":24,"value":235},{"type":19,"tag":70,"props":532,"children":533},{"style":89},[534],{"type":24,"value":240},{"type":19,"tag":70,"props":536,"children":537},{"style":83},[538],{"type":24,"value":245},{"type":19,"tag":70,"props":540,"children":541},{"style":89},[542],{"type":24,"value":250},{"type":19,"tag":70,"props":544,"children":545},{"style":253},[546],{"type":24,"value":256},{"type":19,"tag":70,"props":548,"children":549},{"style":259},[550],{"type":24,"value":262},{"type":19,"tag":70,"props":552,"children":553},{"style":253},[554],{"type":24,"value":267},{"type":19,"tag":70,"props":556,"children":557},{"style":83},[558],{"type":24,"value":136},{"type":19,"tag":70,"props":560,"children":561},{"class":72,"line":191},[562,566,570,574,578,583,587,592],{"type":19,"tag":70,"props":563,"children":564},{"style":83},[565],{"type":24,"value":280},{"type":19,"tag":70,"props":567,"children":568},{"style":89},[569],{"type":24,"value":240},{"type":19,"tag":70,"props":571,"children":572},{"style":83},[573],{"type":24,"value":289},{"type":19,"tag":70,"props":575,"children":576},{"style":89},[577],{"type":24,"value":294},{"type":19,"tag":70,"props":579,"children":580},{"style":118},[581],{"type":24,"value":582}," WindowEventTrigger",{"type":19,"tag":70,"props":584,"children":585},{"style":83},[586],{"type":24,"value":126},{"type":19,"tag":70,"props":588,"children":589},{"style":118},[590],{"type":24,"value":591},"Seq",{"type":19,"tag":70,"props":593,"children":594},{"style":83},[595],{"type":24,"value":596},"(m.timestamp, m.sourceIp), messageLimit))\n",{"type":19,"tag":70,"props":598,"children":599},{"class":72,"line":201},[600],{"type":19,"tag":70,"props":601,"children":602},{"style":83},[603],{"type":24,"value":604},"    .reduceByKeyAndWindow(_ add _, _ remove _, messageWindow)\n",{"type":19,"tag":70,"props":606,"children":607},{"class":72,"line":211},[608],{"type":19,"tag":70,"props":609,"children":610},{"style":83},[611],{"type":24,"value":612},"    .filter(_._2.triggerNow)\n",{"type":19,"tag":70,"props":614,"children":615},{"class":72,"line":220},[616,620,624],{"type":19,"tag":70,"props":617,"children":618},{"style":83},[619],{"type":24,"value":369},{"type":19,"tag":70,"props":621,"children":622},{"style":89},[623],{"type":24,"value":240},{"type":19,"tag":70,"props":625,"children":626},{"style":83},[627],{"type":24,"value":378},{"type":19,"tag":20,"props":629,"children":630},{},[631,633,638],{"type":24,"value":632},"How this works is simple. We have a case class called ",{"type":19,"tag":51,"props":634,"children":636},{"className":635},[],[637],{"type":24,"value":492},{"type":24,"value":639}," that we map into the stream for each incoming message. It then:",{"type":19,"tag":461,"props":641,"children":642},{},[643,648],{"type":19,"tag":434,"props":644,"children":645},{},[646],{"type":24,"value":647},"Tracks incoming messages - if it hits the level, sets the flag, and notes the event",{"type":19,"tag":434,"props":649,"children":650},{},[651],{"type":24,"value":652},"Tracks outgoing messages - and resets when the event that caused the trigger leaves the window",{"type":19,"tag":654,"props":655,"children":656},"blockquote",{},[657],{"type":19,"tag":20,"props":658,"children":659},{},[660,662,667],{"type":24,"value":661},"By switching to the in-memory ",{"type":19,"tag":51,"props":663,"children":665},{"className":664},[],[666],{"type":24,"value":46},{"type":24,"value":668},", Spark needs to persist state in case executors go down or it is necessary to shuffle data between them. Ensure your SparkStreamingContext object has a checkpoint folder set to reliable storage like HDFS.",{"type":19,"tag":41,"props":670,"children":672},{"id":671},"windoweventtrigger-class",[673],{"type":24,"value":674},"WindowEventTrigger class",{"type":19,"tag":20,"props":676,"children":677},{},[678,680,685],{"type":24,"value":679},"Here is the ",{"type":19,"tag":51,"props":681,"children":683},{"className":682},[],[684],{"type":24,"value":492},{"type":24,"value":686}," class for your utilisation.",{"type":19,"tag":59,"props":688,"children":690},{"className":61,"code":689,"language":63,"meta":64,"style":64},"case class WindowEventTrigger[T] private(eventsInWindow: Seq[T], triggerNow: Boolean, private val lastTriggeredEvent: Option[T], private val triggerLevel: Int) {\n  def this(item: T, triggerLevel: Int) = this(Seq(item), false, None, triggerLevel)\n\n  def add(incoming: WindowEventTrigger[T]): WindowEventTrigger[T] = {\n    val combined = eventsInWindow ++ incoming.eventsInWindow\n    val shouldTrigger = lastTriggeredEvent.isEmpty && combined.length >= triggerLevel\n    val triggeredEvent = if (shouldTrigger) combined.seq.drop(triggerLevel - 1).headOption else lastTriggeredEvent\n    new WindowEventTrigger(combined, shouldTrigger, triggeredEvent, triggerLevel)\n  }\n\n  def remove(outgoing: WindowEventTrigger[T]): WindowEventTrigger[T] = {\n    val reduced = eventsInWindow.filterNot(y => outgoing.eventsInWindow.contains(y))\n    val triggeredEvent = if (lastTriggeredEvent.isDefined && outgoing.eventsInWindow.contains(lastTriggeredEvent.get)) None else lastTriggeredEvent\n    new WindowEventTrigger(reduced, false, triggeredEvent, triggerLevel)\n  }\n}\n",[691],{"type":19,"tag":51,"props":692,"children":693},{"__ignoreMap":64},[694,844,940,947,1018,1049,1089,1140,1157,1165,1172,1241,1271,1317,1342,1349],{"type":19,"tag":70,"props":695,"children":696},{"class":72,"line":73},[697,702,707,711,716,721,726,731,735,741,746,750,754,758,763,767,771,776,781,785,790,795,800,805,809,813,817,821,825,830,834,839],{"type":19,"tag":70,"props":698,"children":699},{"style":77},[700],{"type":24,"value":701},"case",{"type":19,"tag":70,"props":703,"children":704},{"style":77},[705],{"type":24,"value":706}," class",{"type":19,"tag":70,"props":708,"children":709},{"style":118},[710],{"type":24,"value":582},{"type":19,"tag":70,"props":712,"children":713},{"style":83},[714],{"type":24,"value":715},"[",{"type":19,"tag":70,"props":717,"children":718},{"style":118},[719],{"type":24,"value":720},"T",{"type":19,"tag":70,"props":722,"children":723},{"style":83},[724],{"type":24,"value":725},"] ",{"type":19,"tag":70,"props":727,"children":728},{"style":89},[729],{"type":24,"value":730},"private",{"type":19,"tag":70,"props":732,"children":733},{"style":83},[734],{"type":24,"value":126},{"type":19,"tag":70,"props":736,"children":738},{"style":737},"--shiki-default:#5C6A72;--shiki-default-font-style:inherit;--shiki-dark:#FFB86C;--shiki-dark-font-style:italic",[739],{"type":24,"value":740},"eventsInWindow",{"type":19,"tag":70,"props":742,"children":743},{"style":83},[744],{"type":24,"value":745},": ",{"type":19,"tag":70,"props":747,"children":748},{"style":118},[749],{"type":24,"value":591},{"type":19,"tag":70,"props":751,"children":752},{"style":83},[753],{"type":24,"value":715},{"type":19,"tag":70,"props":755,"children":756},{"style":118},[757],{"type":24,"value":720},{"type":19,"tag":70,"props":759,"children":760},{"style":83},[761],{"type":24,"value":762},"], ",{"type":19,"tag":70,"props":764,"children":765},{"style":737},[766],{"type":24,"value":500},{"type":19,"tag":70,"props":768,"children":769},{"style":83},[770],{"type":24,"value":745},{"type":19,"tag":70,"props":772,"children":773},{"style":118},[774],{"type":24,"value":775},"Boolean",{"type":19,"tag":70,"props":777,"children":778},{"style":83},[779],{"type":24,"value":780},", ",{"type":19,"tag":70,"props":782,"children":783},{"style":89},[784],{"type":24,"value":730},{"type":19,"tag":70,"props":786,"children":787},{"style":89},[788],{"type":24,"value":789}," val",{"type":19,"tag":70,"props":791,"children":792},{"style":83},[793],{"type":24,"value":794}," lastTriggeredEvent",{"type":19,"tag":70,"props":796,"children":797},{"style":89},[798],{"type":24,"value":799},":",{"type":19,"tag":70,"props":801,"children":802},{"style":118},[803],{"type":24,"value":804}," Option",{"type":19,"tag":70,"props":806,"children":807},{"style":83},[808],{"type":24,"value":715},{"type":19,"tag":70,"props":810,"children":811},{"style":118},[812],{"type":24,"value":720},{"type":19,"tag":70,"props":814,"children":815},{"style":83},[816],{"type":24,"value":762},{"type":19,"tag":70,"props":818,"children":819},{"style":89},[820],{"type":24,"value":730},{"type":19,"tag":70,"props":822,"children":823},{"style":89},[824],{"type":24,"value":789},{"type":19,"tag":70,"props":826,"children":827},{"style":83},[828],{"type":24,"value":829}," triggerLevel",{"type":19,"tag":70,"props":831,"children":832},{"style":89},[833],{"type":24,"value":799},{"type":19,"tag":70,"props":835,"children":836},{"style":118},[837],{"type":24,"value":838}," Int",{"type":19,"tag":70,"props":840,"children":841},{"style":83},[842],{"type":24,"value":843},") {\n",{"type":19,"tag":70,"props":845,"children":846},{"class":72,"line":101},[847,852,858,862,867,871,875,879,884,888,893,898,902,907,911,915,920,926,930,935],{"type":19,"tag":70,"props":848,"children":849},{"style":77},[850],{"type":24,"value":851},"  def",{"type":19,"tag":70,"props":853,"children":855},{"style":854},"--shiki-default:#8DA101;--shiki-dark:#50FA7B",[856],{"type":24,"value":857}," this",{"type":19,"tag":70,"props":859,"children":860},{"style":83},[861],{"type":24,"value":126},{"type":19,"tag":70,"props":863,"children":864},{"style":737},[865],{"type":24,"value":866},"item",{"type":19,"tag":70,"props":868,"children":869},{"style":83},[870],{"type":24,"value":745},{"type":19,"tag":70,"props":872,"children":873},{"style":118},[874],{"type":24,"value":720},{"type":19,"tag":70,"props":876,"children":877},{"style":83},[878],{"type":24,"value":780},{"type":19,"tag":70,"props":880,"children":881},{"style":737},[882],{"type":24,"value":883},"triggerLevel",{"type":19,"tag":70,"props":885,"children":886},{"style":83},[887],{"type":24,"value":745},{"type":19,"tag":70,"props":889,"children":890},{"style":118},[891],{"type":24,"value":892},"Int",{"type":19,"tag":70,"props":894,"children":895},{"style":83},[896],{"type":24,"value":897},") ",{"type":19,"tag":70,"props":899,"children":900},{"style":89},[901],{"type":24,"value":92},{"type":19,"tag":70,"props":903,"children":905},{"style":904},"--shiki-default:#5C6A72;--shiki-default-font-style:inherit;--shiki-dark:#BD93F9;--shiki-dark-font-style:italic",[906],{"type":24,"value":857},{"type":19,"tag":70,"props":908,"children":909},{"style":83},[910],{"type":24,"value":126},{"type":19,"tag":70,"props":912,"children":913},{"style":118},[914],{"type":24,"value":591},{"type":19,"tag":70,"props":916,"children":917},{"style":83},[918],{"type":24,"value":919},"(item), ",{"type":19,"tag":70,"props":921,"children":923},{"style":922},"--shiki-default:#3A94C5;--shiki-dark:#BD93F9",[924],{"type":24,"value":925},"false",{"type":19,"tag":70,"props":927,"children":928},{"style":83},[929],{"type":24,"value":780},{"type":19,"tag":70,"props":931,"children":932},{"style":118},[933],{"type":24,"value":934},"None",{"type":19,"tag":70,"props":936,"children":937},{"style":83},[938],{"type":24,"value":939},", triggerLevel)\n",{"type":19,"tag":70,"props":941,"children":942},{"class":72,"line":139},[943],{"type":19,"tag":70,"props":944,"children":945},{"emptyLinePlaceholder":195},[946],{"type":24,"value":198},{"type":19,"tag":70,"props":948,"children":949},{"class":72,"line":191},[950,954,959,963,968,972,976,980,984,989,993,997,1001,1005,1009,1013],{"type":19,"tag":70,"props":951,"children":952},{"style":77},[953],{"type":24,"value":851},{"type":19,"tag":70,"props":955,"children":956},{"style":854},[957],{"type":24,"value":958}," add",{"type":19,"tag":70,"props":960,"children":961},{"style":83},[962],{"type":24,"value":126},{"type":19,"tag":70,"props":964,"children":965},{"style":737},[966],{"type":24,"value":967},"incoming",{"type":19,"tag":70,"props":969,"children":970},{"style":83},[971],{"type":24,"value":745},{"type":19,"tag":70,"props":973,"children":974},{"style":118},[975],{"type":24,"value":492},{"type":19,"tag":70,"props":977,"children":978},{"style":83},[979],{"type":24,"value":715},{"type":19,"tag":70,"props":981,"children":982},{"style":118},[983],{"type":24,"value":720},{"type":19,"tag":70,"props":985,"children":986},{"style":83},[987],{"type":24,"value":988},"])",{"type":19,"tag":70,"props":990,"children":991},{"style":89},[992],{"type":24,"value":799},{"type":19,"tag":70,"props":994,"children":995},{"style":118},[996],{"type":24,"value":582},{"type":19,"tag":70,"props":998,"children":999},{"style":83},[1000],{"type":24,"value":715},{"type":19,"tag":70,"props":1002,"children":1003},{"style":118},[1004],{"type":24,"value":720},{"type":19,"tag":70,"props":1006,"children":1007},{"style":83},[1008],{"type":24,"value":725},{"type":19,"tag":70,"props":1010,"children":1011},{"style":89},[1012],{"type":24,"value":92},{"type":19,"tag":70,"props":1014,"children":1015},{"style":83},[1016],{"type":24,"value":1017}," {\n",{"type":19,"tag":70,"props":1019,"children":1020},{"class":72,"line":201},[1021,1026,1031,1035,1040,1044],{"type":19,"tag":70,"props":1022,"children":1023},{"style":89},[1024],{"type":24,"value":1025},"    val",{"type":19,"tag":70,"props":1027,"children":1028},{"style":83},[1029],{"type":24,"value":1030}," combined ",{"type":19,"tag":70,"props":1032,"children":1033},{"style":89},[1034],{"type":24,"value":92},{"type":19,"tag":70,"props":1036,"children":1037},{"style":83},[1038],{"type":24,"value":1039}," eventsInWindow ",{"type":19,"tag":70,"props":1041,"children":1042},{"style":89},[1043],{"type":24,"value":327},{"type":19,"tag":70,"props":1045,"children":1046},{"style":83},[1047],{"type":24,"value":1048}," incoming.eventsInWindow\n",{"type":19,"tag":70,"props":1050,"children":1051},{"class":72,"line":211},[1052,1056,1061,1065,1070,1075,1080,1084],{"type":19,"tag":70,"props":1053,"children":1054},{"style":89},[1055],{"type":24,"value":1025},{"type":19,"tag":70,"props":1057,"children":1058},{"style":83},[1059],{"type":24,"value":1060}," shouldTrigger ",{"type":19,"tag":70,"props":1062,"children":1063},{"style":89},[1064],{"type":24,"value":92},{"type":19,"tag":70,"props":1066,"children":1067},{"style":83},[1068],{"type":24,"value":1069}," lastTriggeredEvent.isEmpty ",{"type":19,"tag":70,"props":1071,"children":1072},{"style":89},[1073],{"type":24,"value":1074},"&&",{"type":19,"tag":70,"props":1076,"children":1077},{"style":83},[1078],{"type":24,"value":1079}," combined.length ",{"type":19,"tag":70,"props":1081,"children":1082},{"style":89},[1083],{"type":24,"value":355},{"type":19,"tag":70,"props":1085,"children":1086},{"style":83},[1087],{"type":24,"value":1088}," triggerLevel\n",{"type":19,"tag":70,"props":1090,"children":1091},{"class":72,"line":220},[1092,1096,1101,1105,1110,1115,1120,1125,1130,1135],{"type":19,"tag":70,"props":1093,"children":1094},{"style":89},[1095],{"type":24,"value":1025},{"type":19,"tag":70,"props":1097,"children":1098},{"style":83},[1099],{"type":24,"value":1100}," triggeredEvent ",{"type":19,"tag":70,"props":1102,"children":1103},{"style":89},[1104],{"type":24,"value":92},{"type":19,"tag":70,"props":1106,"children":1107},{"style":77},[1108],{"type":24,"value":1109}," if",{"type":19,"tag":70,"props":1111,"children":1112},{"style":83},[1113],{"type":24,"value":1114}," (shouldTrigger) combined.seq.drop(triggerLevel ",{"type":19,"tag":70,"props":1116,"children":1117},{"style":89},[1118],{"type":24,"value":1119},"-",{"type":19,"tag":70,"props":1121,"children":1122},{"style":95},[1123],{"type":24,"value":1124}," 1",{"type":19,"tag":70,"props":1126,"children":1127},{"style":83},[1128],{"type":24,"value":1129},").headOption ",{"type":19,"tag":70,"props":1131,"children":1132},{"style":77},[1133],{"type":24,"value":1134},"else",{"type":19,"tag":70,"props":1136,"children":1137},{"style":83},[1138],{"type":24,"value":1139}," lastTriggeredEvent\n",{"type":19,"tag":70,"props":1141,"children":1142},{"class":72,"line":229},[1143,1148,1152],{"type":19,"tag":70,"props":1144,"children":1145},{"style":77},[1146],{"type":24,"value":1147},"    new",{"type":19,"tag":70,"props":1149,"children":1150},{"style":118},[1151],{"type":24,"value":582},{"type":19,"tag":70,"props":1153,"children":1154},{"style":83},[1155],{"type":24,"value":1156},"(combined, shouldTrigger, triggeredEvent, triggerLevel)\n",{"type":19,"tag":70,"props":1158,"children":1159},{"class":72,"line":274},[1160],{"type":19,"tag":70,"props":1161,"children":1162},{"style":83},[1163],{"type":24,"value":1164},"  }\n",{"type":19,"tag":70,"props":1166,"children":1167},{"class":72,"line":307},[1168],{"type":19,"tag":70,"props":1169,"children":1170},{"emptyLinePlaceholder":195},[1171],{"type":24,"value":198},{"type":19,"tag":70,"props":1173,"children":1174},{"class":72,"line":335},[1175,1179,1184,1188,1193,1197,1201,1205,1209,1213,1217,1221,1225,1229,1233,1237],{"type":19,"tag":70,"props":1176,"children":1177},{"style":77},[1178],{"type":24,"value":851},{"type":19,"tag":70,"props":1180,"children":1181},{"style":854},[1182],{"type":24,"value":1183}," remove",{"type":19,"tag":70,"props":1185,"children":1186},{"style":83},[1187],{"type":24,"value":126},{"type":19,"tag":70,"props":1189,"children":1190},{"style":737},[1191],{"type":24,"value":1192},"outgoing",{"type":19,"tag":70,"props":1194,"children":1195},{"style":83},[1196],{"type":24,"value":745},{"type":19,"tag":70,"props":1198,"children":1199},{"style":118},[1200],{"type":24,"value":492},{"type":19,"tag":70,"props":1202,"children":1203},{"style":83},[1204],{"type":24,"value":715},{"type":19,"tag":70,"props":1206,"children":1207},{"style":118},[1208],{"type":24,"value":720},{"type":19,"tag":70,"props":1210,"children":1211},{"style":83},[1212],{"type":24,"value":988},{"type":19,"tag":70,"props":1214,"children":1215},{"style":89},[1216],{"type":24,"value":799},{"type":19,"tag":70,"props":1218,"children":1219},{"style":118},[1220],{"type":24,"value":582},{"type":19,"tag":70,"props":1222,"children":1223},{"style":83},[1224],{"type":24,"value":715},{"type":19,"tag":70,"props":1226,"children":1227},{"style":118},[1228],{"type":24,"value":720},{"type":19,"tag":70,"props":1230,"children":1231},{"style":83},[1232],{"type":24,"value":725},{"type":19,"tag":70,"props":1234,"children":1235},{"style":89},[1236],{"type":24,"value":92},{"type":19,"tag":70,"props":1238,"children":1239},{"style":83},[1240],{"type":24,"value":1017},{"type":19,"tag":70,"props":1242,"children":1243},{"class":72,"line":363},[1244,1248,1253,1257,1262,1266],{"type":19,"tag":70,"props":1245,"children":1246},{"style":89},[1247],{"type":24,"value":1025},{"type":19,"tag":70,"props":1249,"children":1250},{"style":83},[1251],{"type":24,"value":1252}," reduced ",{"type":19,"tag":70,"props":1254,"children":1255},{"style":89},[1256],{"type":24,"value":92},{"type":19,"tag":70,"props":1258,"children":1259},{"style":83},[1260],{"type":24,"value":1261}," eventsInWindow.filterNot(y ",{"type":19,"tag":70,"props":1263,"children":1264},{"style":89},[1265],{"type":24,"value":240},{"type":19,"tag":70,"props":1267,"children":1268},{"style":83},[1269],{"type":24,"value":1270}," outgoing.eventsInWindow.contains(y))\n",{"type":19,"tag":70,"props":1272,"children":1273},{"class":72,"line":381},[1274,1278,1282,1286,1290,1295,1299,1304,1308,1313],{"type":19,"tag":70,"props":1275,"children":1276},{"style":89},[1277],{"type":24,"value":1025},{"type":19,"tag":70,"props":1279,"children":1280},{"style":83},[1281],{"type":24,"value":1100},{"type":19,"tag":70,"props":1283,"children":1284},{"style":89},[1285],{"type":24,"value":92},{"type":19,"tag":70,"props":1287,"children":1288},{"style":77},[1289],{"type":24,"value":1109},{"type":19,"tag":70,"props":1291,"children":1292},{"style":83},[1293],{"type":24,"value":1294}," (lastTriggeredEvent.isDefined ",{"type":19,"tag":70,"props":1296,"children":1297},{"style":89},[1298],{"type":24,"value":1074},{"type":19,"tag":70,"props":1300,"children":1301},{"style":83},[1302],{"type":24,"value":1303}," outgoing.eventsInWindow.contains(lastTriggeredEvent.get)) ",{"type":19,"tag":70,"props":1305,"children":1306},{"style":118},[1307],{"type":24,"value":934},{"type":19,"tag":70,"props":1309,"children":1310},{"style":77},[1311],{"type":24,"value":1312}," else",{"type":19,"tag":70,"props":1314,"children":1315},{"style":83},[1316],{"type":24,"value":1139},{"type":19,"tag":70,"props":1318,"children":1319},{"class":72,"line":389},[1320,1324,1328,1333,1337],{"type":19,"tag":70,"props":1321,"children":1322},{"style":77},[1323],{"type":24,"value":1147},{"type":19,"tag":70,"props":1325,"children":1326},{"style":118},[1327],{"type":24,"value":582},{"type":19,"tag":70,"props":1329,"children":1330},{"style":83},[1331],{"type":24,"value":1332},"(reduced, ",{"type":19,"tag":70,"props":1334,"children":1335},{"style":922},[1336],{"type":24,"value":925},{"type":19,"tag":70,"props":1338,"children":1339},{"style":83},[1340],{"type":24,"value":1341},", triggeredEvent, triggerLevel)\n",{"type":19,"tag":70,"props":1343,"children":1344},{"class":72,"line":398},[1345],{"type":19,"tag":70,"props":1346,"children":1347},{"style":83},[1348],{"type":24,"value":1164},{"type":19,"tag":70,"props":1350,"children":1352},{"class":72,"line":1351},16,[1353],{"type":19,"tag":70,"props":1354,"children":1355},{"style":83},[1356],{"type":24,"value":1357},"}\n",{"type":19,"tag":20,"props":1359,"children":1360},{},[1361],{"type":24,"value":1362},"Happy streaming,",{"type":19,"tag":20,"props":1364,"children":1365},{},[1366],{"type":19,"tag":1367,"props":1368,"children":1369},"em",{},[1370],{"type":24,"value":1371},"[)amien",{"type":19,"tag":1373,"props":1374,"children":1375},"style",{},[1376],{"type":24,"value":1377},"html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":64,"searchDepth":101,"depth":101,"links":1379},[1380,1381,1382,1383],{"id":43,"depth":101,"text":46},{"id":407,"depth":101,"text":410},{"id":451,"depth":101,"text":454},{"id":671,"depth":101,"text":674},"markdown","content:blog:2015:time-window-events-with-apache-spark-streaming.md","content","blog/2015/time-window-events-with-apache-spark-streaming.md","blog/2015/time-window-events-with-apache-spark-streaming","md","/blog/2015/time-window-events-with-apache-spark-streaming/",661,[1393,1397,1401],{"title":1394,"date":1395,"url":1396},"HTML5 Video Cheatsheet: Optimizing videos for the web","2025-12-05T00:00:00Z","/blog/2025/html5-video-cheatsheet/",{"title":1398,"date":1399,"url":1400},"Transactions in the MongoDB EF Core Provider","2025-10-25","/blog/2025/mongodb-explicit-transactions/",{"title":1402,"date":1403,"url":1404},"Queryable Encryption with the MongoDB EF Core Provider","2025-09-22","/blog/2025/mongodb-queryable-encryption/",[1406],{"_path":1407,"_dir":1408,"_draft":6,"_partial":6,"_locale":7,"title":1409,"description":1410,"id":1411,"name":1412,"email":1413,"avatar":1414,"date":1415,"body":1416,"_type":1384,"_id":1424,"_source":1386,"_file":1425,"_stem":1426,"_extension":1389},"/comments/time-window-events-with-apache-spark-streaming/156542","time-window-events-with-apache-spark-streaming","156542","Have you had a chance to use the other windowed functions like Lag on a  Dstream? We are experiencing performance issues even with really low data volumes and it doesn't seem like the right solution for calculating the time duration from the current record to the previous for matching fields. At least not in 1.4.1.",156542,"Scott F","scott.faculak@gmail.com","https://www.gravatar.com/avatar/265ea007b95719965c6ec49e0ac43599?r=pg&d=retro","2015-12-18T19:47:35",{"type":16,"children":1417,"toc":1422},[1418],{"type":19,"tag":20,"props":1419,"children":1420},{},[1421],{"type":24,"value":1410},{"title":64,"searchDepth":101,"depth":101,"links":1423},[],"content:comments:time-window-events-with-apache-spark-streaming:156542.md","comments/time-window-events-with-apache-spark-streaming/156542.md","comments/time-window-events-with-apache-spark-streaming/156542",1779264582271]