Add Rake Tasks for Fetching Past Events (#1064)

Author: marcoroth

Rakefile

@@ -85,6 +85,14 @@ task :fetch_meetups do
   end
 end
 
+desc "fetch past meetups"
+task :fetch_past_meetups do
+  MeetupsFile.read.tap do |file|
+    file.fetch!(past: true)
+    file.write!
+  end
+end
+
 # to fetch a single group run:
 #   bundle exec rake fetch_meetup[sfruby]
 desc "fetch a single group"
@@ -95,6 +103,16 @@ task :fetch_meetup, [:group_id] do |_, args|
   end
 end
 
+# to fetch past events of a single group run:
+#   bundle exec rake fetch_past_meetups[sfruby]
+desc "fetch past meetups of a single group"
+task :fetch_past_meetup, [:group_id] do |_, args|
+  MeetupsFile.read.tap do |file|
+    file.fetch!(args[:group_id], past: true)
+    file.write!
+  end
+end
+
 desc "fetch write all meetup groups"
 task :fetch_and_write_all_meetups do
   MeetupGroup.all.each do |group|

src/events/meetup_event.rb

@@ -20,7 +20,10 @@ def event_url
   end
 
   def event_status
-    object.status == "published" ? nil : object.status
+    return nil if object.status&.downcase == "published"
+    return nil if object.status&.downcase == "past"
+
+    object.status
   end
 
   def event_location

src/meetup_graphql_schema.json

@@ -82,6 +82,36 @@
               },
               "args": []
             },
+            {
+              "name": "pastEvents",
+              "type": {
+                "kind": "OBJECT",
+                "name": "PastEventConnection"
+              },
+              "args": [
+                {
+                  "name": "sortOrder",
+                  "type": {
+                    "kind": "ENUM",
+                    "name": "SortOrder"
+                  }
+                },
+                {
+                  "name": "input",
+                  "type": {
+                    "kind": "INPUT_OBJECT",
+                    "name": "ConnectionInput"
+                  }
+                },
+                {
+                  "name": "filter",
+                  "type": {
+                    "kind": "INPUT_OBJECT",
+                    "name": "GroupUpcomingEventsFilter"
+                  }
+                }
+              ]
+            },
             {
               "name": "upcomingEvents",
               "type": {
@@ -152,6 +182,31 @@
             }
           ]
         },
+        {
+          "kind": "OBJECT",
+          "name": "PastEventConnection",
+          "fields": [
+            {
+              "name": "count",
+              "type": {
+                "kind": "SCALAR",
+                "name": "Int"
+              },
+              "args": []
+            },
+            {
+              "name": "edges",
+              "type": {
+                "kind": "LIST",
+                "ofType": {
+                  "kind": "OBJECT",
+                  "name": "PastEventEdge"
+                }
+              },
+              "args": []
+            }
+          ]
+        },
         {
           "kind": "OBJECT",
           "name": "EventConnection",
@@ -205,6 +260,28 @@
             }
           ]
         },
+        {
+          "kind": "OBJECT",
+          "name": "PastEventEdge",
+          "fields": [
+            {
+              "name": "cursor",
+              "type": {
+                "kind": "SCALAR",
+                "name": "String"
+              },
+              "args": []
+            },
+            {
+              "name": "node",
+              "type": {
+                "kind": "OBJECT",
+                "name": "Event"
+              },
+              "args": []
+            }
+          ]
+        },
         {
           "kind": "OBJECT",
           "name": "EventEdge",

src/meetups_file.rb

@@ -32,7 +32,7 @@ def find_by(service_id: nil, url: nil)
     end
   end
 
-  def fetch_group!(group)
+  def fetch_group!(group, past: false)
     puts "Fetching #{group.service} Group: #{group.id}"
 
     group.new_events.map { |event| event.meetup_file_entry }.each do |event|
@@ -41,6 +41,14 @@ def fetch_group!(group)
       puts "New Meetup: #{event.name} - #{event.date}"
     end
 
+    if past
+      group.new_past_events.map { |event| event.meetup_file_entry }.each do |event|
+        @new_events << event
+        @events << event
+        puts "New Past Meetup: #{event.name} - #{event.date}"
+      end
+    end
+
     group.cancelled_events.map { |event| event.meetup_file_entry }.each do |event|
       event_entry = find_by(url: event.url)
 
@@ -70,11 +78,11 @@ def fetch_group!(group)
     puts
   end
 
-  def fetch!(id = nil)
+  def fetch!(id = nil, past: false)
     groups = MeetupGroup.all
     groups = groups.where(id: id) if id
 
-    groups.each { |group| fetch_group!(group) }
+    groups.each { |group| fetch_group!(group, past: past) }
 
     puts "New Events: #{@new_events.count}"
     puts "Updated Events: #{@updated_events.count}"

src/queries/events_query.rb

@@ -1,5 +1,49 @@
 require_relative "../meetup_client"
 
+eventConnection = <<-GRAPHQL
+  count
+  edges {
+    cursor
+    node {
+      id
+      title
+      eventUrl
+      shortDescription
+      description
+      onlineVenue {
+        type
+        url
+      }
+      venue {
+        address
+        city
+        state
+        country
+      }
+      host {
+        id
+        name
+        email
+      }
+      status
+      dateTime
+      endTime
+      duration
+      timezone
+      createdAt
+      eventType
+      isOnline
+      group {
+        id
+        name
+        country
+        state
+        city
+      }
+    }
+  }
+GRAPHQL
+
 EventsQuery = MeetupClient::Client.parse(<<-GRAPHQL
   query ($groupId: String!) {
     groupByUrlname(urlname: $groupId) {
@@ -15,47 +59,29 @@
       state
       city
       upcomingEvents(input: { first: 50 }, filter: { includeCancelled: true }, sortOrder: ASC) {
-        count
-        edges {
-          cursor
-          node {
-            id
-            title
-            eventUrl
-            shortDescription
-            description
-            onlineVenue {
-              type
-              url
-            }
-            venue {
-              address
-              city
-              state
-              country
-            }
-            host {
-              id
-              name
-              email
-            }
-            status
-            dateTime
-            endTime
-            duration
-            timezone
-            createdAt
-            eventType
-            isOnline
-            group {
-              id
-              name
-              country
-              state
-              city
-            }
-          }
-        }
+        #{eventConnection}
+      }
+    }
+  }
+  GRAPHQL
+)
+
+PastEventsQuery = MeetupClient::Client.parse(<<-GRAPHQL
+  query ($groupId: String!) {
+    groupByUrlname(urlname: $groupId) {
+      id
+      logo {
+        id
+        baseUrl
+        preview
+        source
+      }
+      name
+      country
+      state
+      city
+      pastEvents(input: { first: 1000 }, sortOrder: ASC){
+        #{eventConnection}
       }
     }
   }

src/static/meetup_group.rb

@@ -37,6 +37,20 @@ def new_events
     upcoming_events.select { |event| !existing_ids.include?(event.service_id) }
   end
 
+  def new_past_events
+    existing_ids = past_existing_events.map(&:service_id)
+
+    past_events.select { |event| !existing_ids.include?(event.service_id) }
+  end
+
+  def past_events
+    @past_events ||= fetch_past_events.tap do |events|
+      events.select! { |event| event.event_name.include?(filter) } if filter.present?
+      events.reject! { |event| Array(exclude).any? { |e| event.event_name.include?(e) } } if exclude.present?
+      events.sort_by { |event| [event.event_date, event.event_name] }
+    end
+  end
+
   def missing_events
     upcoming_ids = upcoming_events.map(&:service_id)
 
@@ -47,6 +61,10 @@ def upcomping_existing_events
     existing_events.select { |event| event["date"].between?(Date.today - 1, Date.today + 120) }
   end
 
+  def past_existing_events
+    existing_events.select { |event| event["date"] < Date.today }
+  end
+
   def existing_events
     Meetup.for_group(self)
   end
@@ -69,6 +87,19 @@ def tz
 
   private
 
+  def fetch_past_events
+    case service
+    when "meetupdotcom"
+      fetch_past_meetup_events
+    when "luma"
+      fetch_luma_events.select { |event| event.date < Date.today }
+    when "ical"
+      fetch_ical_events.select { |event| event.date < Date.today }
+    else
+      raise "Unsupported service: #{service}"
+    end
+  end
+
   def fetch_events
     case service
     when "meetupdotcom"
@@ -82,6 +113,12 @@ def fetch_events
     end
   end
 
+  def fetch_past_meetup_events
+    result = MeetupClient::Client.query(PastEventsQuery, variables: { groupId: id })
+    events = Array(result.original_hash.dig("data", "groupByUrlname", "pastEvents", "edges"))
+    events.map { |event| MeetupEvent.new(object: OpenStruct.new(event["node"]), group: self) }
+  end
+
   def fetch_meetup_events
     result = MeetupClient::Client.query(EventsQuery, variables: { groupId: id })
     events = Array(result.original_hash.dig("data", "groupByUrlname", "upcomingEvents", "edges"))